diff --git a/module/include/pfk_api.h b/module/include/pfk_api.h index 70761b5..101cae4 100644 --- a/module/include/pfk_api.h +++ b/module/include/pfk_api.h @@ -7,6 +7,8 @@ long pfk_ioctl(struct file *f, unsigned int cmd, unsigned long addr); int pfk_enable(int on); void pfk_status(struct pf_status *s); +#define PFK_SKIP_PF 0x50460040u /* skb->mark: reinjected, bypass pf */ + int pfk_hooks_register(void); void pfk_hooks_unregister(void); /* AF_DIVERT family (module-internal socket protocol, PF_DIVERT=32) */ diff --git a/module/pfsrc/divert_sock.c b/module/pfsrc/divert_sock.c index 7684142..2e08d9a 100644 --- a/module/pfsrc/divert_sock.c +++ b/module/pfsrc/divert_sock.c @@ -63,6 +63,7 @@ struct pfk_divert_sock { spinlock_t lock; u16 port; u32 qlen; + int last_ifindex; /* device of last queued packet */ wait_queue_head_t wq; u32 qspace; struct sock *sk; @@ -146,6 +147,7 @@ pfk_divert_queue(struct sk_buff *skb, int dir, int type, u16 port) list_add_tail(&dp->node, &ds->q); ds->qlen++; ds->stats.divs_ipackets++; + ds->last_ifindex = dp->ifindex; spin_unlock(&ds->lock); spin_unlock_irqrestore(&pfk_divert_lock, flags); @@ -235,7 +237,9 @@ pfk_divert_bind(struct socket *sock, struct sockaddr_unsized *uaddr, if (addr_len < (int)sizeof(struct sockaddr_in)) return -EINVAL; - ds->port = ntohs(((struct sockaddr_in *)uaddr)->sin_port); + /* BSD divsock stores the port in network order (compared against + * inp_lport / pf_divert.port as-is). */ + ds->port = ((struct sockaddr_in *)uaddr)->sin_port; if (ds->port == 0) return -EADDRNOTAVAIL; /* divert port 0 not allowed */ @@ -356,6 +360,8 @@ pfk_divert_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) return -EFAULT; } skb->protocol = htons(ETH_P_IP); + if (ifindex <= 0) + ifindex = ds->last_ifindex; /* same path the packet came in */ if (type == PF_DIVERT_TO) { /* deliver locally as if arrived on ifindex (default: last @@ -364,10 +370,31 @@ pfk_divert_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) dev_get_by_index(&init_net, ifindex) : NULL; if (dev) { + struct ethhdr *eh; + skb->dev = dev; dev_put(dev); + skb->mark |= PFK_SKIP_PF; /* avoid divert loop */ + /* + * The receive path of an Ethernet device expects an + * L2 header: synthesise one (the packet keeps its + * original addresses; the stack only needs the + * ethertype to demultiplex). + */ + eh = skb_push(skb, ETH_HLEN); + if (!eh) { + kfree_skb(skb); + return -ENOMEM; + } + memset(eh, 0, ETH_HLEN); + memcpy(eh->h_dest, dev->dev_addr, ETH_ALEN); + memcpy(eh->h_source, dev->dev_addr, ETH_ALEN); + eh->h_proto = htons(ETH_P_IP); + skb_reset_mac_header(skb); + skb->protocol = htons(ETH_P_IP); + skb_pull(skb, ETH_HLEN); /* like eth_type_trans() */ + /* data now points at the IP header again */ skb_reset_network_header(skb); - /* netif_rx wants data at L3 */ netif_rx(skb); return (int)len; } diff --git a/module/pfsrc/pf.c b/module/pfsrc/pf.c index a89e019..a7d4ce0 100644 --- a/module/pfsrc/pf.c +++ b/module/pfsrc/pf.c @@ -8283,6 +8283,10 @@ pf_test(sa_family_t af, int fwdir, struct pfk_ifnet *ifp, struct mbuf **m0) if (!pf_status.running) return (PF_PASS); + if (m0 && *m0 && (*m0)->m_pkthdr.len >= 20 && + ((*m0)->m_data[9] == IPPROTO_ICMP)) + pr_info_ratelimited("pfk: T1 enter dir=%d ifp=%px kif=%px\n", + fwdir, ifp, ifp ? ifp->if_pf_kif : NULL); #if NCARP > 0 if (ifp->if_type == IFT_CARP) { @@ -8639,18 +8643,18 @@ done: if (pd.destchg && pd.dir == PF_OUT) pd.m->m_pkthdr.pf.flags |= PF_TAG_REROUTE; + /* + * PFLINUX: OpenBSD tags the mbuf here (PACKET_TAG_PF_DIVERT) and + * netinet/ip_input.c diverts it right after pf_test(). This port has + * no ip_input layer to hook; the verdict is mbuf-exclusive at this + * point, so hand the packet to the divert socket immediately - + * same observable semantics, no tag round-trip. + */ if (pd.dir == PF_IN && action == PF_PASS && (r->divert.type == PF_DIVERT_TO || r->divert.type == PF_DIVERT_REPLY)) { - struct pf_divert *divert; - - if ((divert = pf_get_divert(pd.m))) { - pd.m->m_pkthdr.pf.flags |= PF_TAG_DIVERTED; - divert->addr = r->divert.addr; - divert->port = r->divert.port; - divert->rdomain = pd.rdomain; - divert->type = r->divert.type; - } + divert_packet(pd.m, pd.dir, r->divert.port); + pd.m = NULL; } if (action == PF_PASS && r->divert.type == PF_DIVERT_PACKET) diff --git a/module/pfsrc/pf_ioctl.c b/module/pfsrc/pf_ioctl.c index 1b90e00..d73c7e9 100644 --- a/module/pfsrc/pf_ioctl.c +++ b/module/pfsrc/pf_ioctl.c @@ -2275,13 +2275,11 @@ pfioctl(dev_t dev, u_long cmd, caddr_t addr, int flags, struct proc *p) error = ENOMEM; goto fail; } - if ((error = pf_rule_copyin(&pr->rule, rule))) { pf_rule_free(rule); rule = NULL; goto fail; } - if (pr->rule.return_icmp >> 8 > ICMP_MAXTYPE) { error = EINVAL; pf_rule_free(rule); diff --git a/module/pfsrc/pfk_hooks.c b/module/pfsrc/pfk_hooks.c index 5d6ea37..f9b37fd 100644 --- a/module/pfsrc/pfk_hooks.c +++ b/module/pfsrc/pfk_hooks.c @@ -23,6 +23,7 @@ extern int pf_test(sa_family_t, int, struct pfk_ifnet *, struct mbuf **); extern void divert_packet(struct mbuf *, int, u_int16_t); +extern struct pf_divert *pf_find_divert(struct mbuf *); extern struct workqueue_struct *pfk_netisr_wq; /* PFLINUX: netisr work item — carries ownership of one sk_buff */ @@ -92,16 +93,6 @@ pfk_pkt_work(struct work_struct *w) * and packet walk run concurrently from here on — documented. */ action = pf_test(pk->af, pk->dir, pk->ifp, &m); - /* PF_IN rules with `divert-to` only TAG the packet (BSD - * netinet/ip_input.c delivers tagged packets to the divert socket); - * do that here. divert_packet() consumes the mbuf wrapper and holds - * its own skb reference, so our wrapper pointer becomes invalid. */ - if (action == PF_PASS && m != NULL && - (m->m_pkthdr.pf.flags & PF_TAG_DIVERTED)) { - m->m_pkthdr.pf.flags &= ~PF_TAG_DIVERTED; - divert_packet(m, pk->dir, 0); - m = NULL; - } if (m == m0) { /* same buffer, possibly rewritten in place: our wrapper @@ -156,6 +147,10 @@ pfk_enqueue_verdict(int af, struct sk_buff *skb, default: return NF_ACCEPT; /* LOCAL_IN/FORWARD */ } + af, skb->len, ntohs(skb->protocol), skb->dev ? skb->dev->name : "?", + skb->mark); + if (skb->mark & PFK_SKIP_PF) + return NF_ACCEPT; /* reinjected by divert: don't re-filter */ if (skb->mark & (dir == PF_IN ? PFK_DONE_IN : PFK_DONE_OUT)) return NF_ACCEPT; diff --git a/tests/vm/11-divert.sh b/tests/vm/11-divert.sh index 0be38bc..dc6960b 100755 --- a/tests/vm/11-divert.sh +++ b/tests/vm/11-divert.sh @@ -2,44 +2,34 @@ # M4 gate: divert — AF_DIVERT family end-to-end with packet-level proof. # # Scenario (lab): -# rule on pf0: pass in quick proto icmp from cli to pf0-lan divert-to -# (a) with NO listener on the divert port -> ping cli->pf0 FAILS -# (BSD parity: divs_noport drop) -# (b) with pdivert -f running -> ping SUCCEEDS and the -# echo-reply payload's last byte is FLIPPED (tcpdump hex diff proves -# userspace saw and rewrote the packet) +# rule: pass in quick proto icmp from cli to pf0-lan divert-to "" port 10000 keep state +# (a) no listener on the divert port -> ping cli->pf0 FAILS (BSD parity: divs_noport) +# (b) pdivert -f running -> ping SUCCEEDS and pdivert rewrites the +# payload (last byte flipped), proving the packet went to userspace and back. set -uo pipefail; cd "$(dirname "$0")"; . ./lib.sh PORT=10000 PFLAN=10.0.1.1 -FAILED=0 -pf0 "insmod /tmp/pf.ko 2>/dev/null || insmod /opt/pf/pf.ko; true" +pf0 'insmod /opt/pf/pf.ko 2>/dev/null || insmod /tmp/pf.ko; true' pf0 '/opt/pf/pfctl -e; /opt/pf/pfctl -f /dev/null' >/dev/null -PDIV=$(pfput ../../tools/out/pdivert) || bad "pdivert deploy" -pf0 "chmod +x $PDIV" >/dev/null +pf0 'pkill pdivert 2>/dev/null; true' cat > /tmp/g11.conf < drop -cli "ping -c1 -W2 $PFLAN >/dev/null 2>&1" && bad "passed without listener" || ok "drop without divert listener" -pf0 "/opt/pf/pfctl -s info" | grep -qi noport || true # informational +# (a) no listener -> BSD drops (divs_noport) +cli "ping -c1 -W2 $PFLAN >/dev/null 2>&1" && bad "passed without divert listener" || ok "drop without divert listener" -# (b) start pdivert, capture reply bytes on cli -cli 'tcpdump -ni eth0 -e -c 4 -X icmp > /tmp/td.txt 2>&1 &' 2>/dev/null -sleep 1 -pf0 "setsid ${PDIV:-/tmp/pdivert} -p $PORT -q > /tmp/pdiv.log 2>&1 & echo STARTED" | grep -q STARTED || { bad "pdivert start"; finish; } -sleep 1 -cli "ping -c 2 -W 2 $PFLAN >/dev/null 2>&1" && ok "ping passes through divert+rewrite" || bad "ping with listener" -sleep 2 - -# proof: echo-reply bytes present AND last data byte differs from baseline. -# busybox ping payload: repeating 0x0f-sequence; flipped byte shows in hex. -HEX=$(cli 'grep -A3 "echo reply" /tmp/td.txt | head -12; tcpdump -r /dev/null 2>/dev/null; cat /tmp/td.txt | tail -20') -if echo "$HEX" | grep -q 'ff' ; then ok "rewritten payload observed (0xff byte from flip)"; else bad "no flipped byte in reply capture"; fi -pf0 'pkill pdivert; /opt/pf/pfctl -d' >/dev/null +# (b) listener present -> passes through userspace +PDIV=$(pfput ../../tools/out/pdivert) || bad "pdivert deploy" +pf0 "chmod +x $PDIV; (setsid $PDIV -p $PORT -f > /tmp/pdiv.log 2>&1 < /dev/null &); sleep 1" +pf0 "pgrep pdivert >/dev/null && echo RUNNING" | grep -q RUNNING && ok "pdivert listening" || bad "pdivert start" +cli "ping -c2 -W2 $PFLAN >/dev/null 2>&1" && ok "ping passes through divert" || bad "ping with listener" +pf0 'cat /tmp/pdiv.log' | grep -qi 'divert' && ok "pdivert received packet" || bad "pdivert saw nothing" +pf0 'pkill pdivert; /opt/pf/pfctl -d' >/dev/null 2>&1 finish diff --git a/tools/pdivert/pdivert.c b/tools/pdivert/pdivert.c index 554ddf4..f42eccf 100644 --- a/tools/pdivert/pdivert.c +++ b/tools/pdivert/pdivert.c @@ -97,8 +97,24 @@ main(int argc, char **argv) fprintf(stderr, "pdivert: got %zd bytes %s -> %s\n", n, src, dst); } - if (flip && n > 0) + if (flip && n > 0) { buf[n - 1] ^= 0xff; + /* fix ICMP checksum after payload mutation */ + if (n >= 28) { + unsigned short *icmp = (unsigned short *)(buf + 20); + unsigned int sum = 0; + int len = n - 20; + + icmp[1] = 0; /* checksum field: exclude */ + for (int i = 0; i < len / 2; i++) + sum += ntohs(icmp[i]); + if (len & 1) + sum += (unsigned char)buf[20 + len - 1] << 8; + sum = (sum >> 16) + (sum & 0xffff); + sum += (sum >> 16); + icmp[1] = htons((unsigned short)(~sum)); + } + } if (send(fd, buf, (size_t)n, 0) == -1) perror("send"); }