divert: end-to-end working (network-order port, direct IN-verdict handoff, L2-synth reinjection, skip-mark anti-loop, pdivert ICMP checksum fix)

module/include/pfk_api.h

@@ -7,6 +7,8 @@
 long pfk_ioctl(struct file *f, unsigned int cmd, unsigned long addr);
 int  pfk_enable(int on);
 void pfk_status(struct pf_status *s);
+#define PFK_SKIP_PF	0x50460040u	/* skb->mark: reinjected, bypass pf */
+
 int  pfk_hooks_register(void);
 void pfk_hooks_unregister(void);
 /* AF_DIVERT family (module-internal socket protocol, PF_DIVERT=32) */

module/pfsrc/divert_sock.c

@@ -63,6 +63,7 @@ struct pfk_divert_sock {
 	spinlock_t		lock;
 	u16			port;
 	u32			qlen;
+	int			last_ifindex;	/* device of last queued packet */
 	wait_queue_head_t	wq;
 	u32			qspace;
 	struct sock		*sk;
@@ -146,6 +147,7 @@ pfk_divert_queue(struct sk_buff *skb, int dir, int type, u16 port)
 	list_add_tail(&dp->node, &ds->q);
 	ds->qlen++;
 	ds->stats.divs_ipackets++;
+	ds->last_ifindex = dp->ifindex;
 	spin_unlock(&ds->lock);
 	spin_unlock_irqrestore(&pfk_divert_lock, flags);
 
@@ -235,7 +237,9 @@ pfk_divert_bind(struct socket *sock, struct sockaddr_unsized *uaddr,
 	if (addr_len < (int)sizeof(struct sockaddr_in))
 		return -EINVAL;
 
-	ds->port = ntohs(((struct sockaddr_in *)uaddr)->sin_port);
+	/* BSD divsock stores the port in network order (compared against
+	 * inp_lport / pf_divert.port as-is). */
+	ds->port = ((struct sockaddr_in *)uaddr)->sin_port;
 	if (ds->port == 0)
 		return -EADDRNOTAVAIL;	/* divert port 0 not allowed */
 
@@ -356,6 +360,8 @@ pfk_divert_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
 		return -EFAULT;
 	}
 	skb->protocol = htons(ETH_P_IP);
+	if (ifindex <= 0)
+		ifindex = ds->last_ifindex;	/* same path the packet came in */
 
 	if (type == PF_DIVERT_TO) {
 		/* deliver locally as if arrived on ifindex (default: last
@@ -364,10 +370,31 @@ pfk_divert_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
 			dev_get_by_index(&init_net, ifindex) : NULL;
 
 		if (dev) {
+			struct ethhdr *eh;
+
 			skb->dev = dev;
 			dev_put(dev);
+			skb->mark |= PFK_SKIP_PF;	/* avoid divert loop */
+			/*
+			 * The receive path of an Ethernet device expects an
+			 * L2 header: synthesise one (the packet keeps its
+			 * original addresses; the stack only needs the
+			 * ethertype to demultiplex).
+			 */
+			eh = skb_push(skb, ETH_HLEN);
+			if (!eh) {
+				kfree_skb(skb);
+				return -ENOMEM;
+			}
+			memset(eh, 0, ETH_HLEN);
+			memcpy(eh->h_dest, dev->dev_addr, ETH_ALEN);
+			memcpy(eh->h_source, dev->dev_addr, ETH_ALEN);
+			eh->h_proto = htons(ETH_P_IP);
+			skb_reset_mac_header(skb);
+			skb->protocol = htons(ETH_P_IP);
+			skb_pull(skb, ETH_HLEN);	/* like eth_type_trans() */
+			/* data now points at the IP header again */
 			skb_reset_network_header(skb);
-			/* netif_rx wants data at L3 */
 			netif_rx(skb);
 			return (int)len;
 		}

module/pfsrc/pf.c

@@ -8283,6 +8283,10 @@ pf_test(sa_family_t af, int fwdir, struct pfk_ifnet *ifp, struct mbuf **m0)
 
 	if (!pf_status.running)
 		return (PF_PASS);
+	if (m0 && *m0 && (*m0)->m_pkthdr.len >= 20 &&
+	    ((*m0)->m_data[9] == IPPROTO_ICMP))
+		pr_info_ratelimited("pfk: T1 enter dir=%d ifp=%px kif=%px\n",
+		    fwdir, ifp, ifp ? ifp->if_pf_kif : NULL);
 
 #if NCARP > 0
 	if (ifp->if_type == IFT_CARP) {
@@ -8639,18 +8643,18 @@ done:
 	if (pd.destchg && pd.dir == PF_OUT)
 		pd.m->m_pkthdr.pf.flags |= PF_TAG_REROUTE;
 
+	/*
+	 * PFLINUX: OpenBSD tags the mbuf here (PACKET_TAG_PF_DIVERT) and
+	 * netinet/ip_input.c diverts it right after pf_test().  This port has
+	 * no ip_input layer to hook; the verdict is mbuf-exclusive at this
+	 * point, so hand the packet to the divert socket immediately -
+	 * same observable semantics, no tag round-trip.
+	 */
 	if (pd.dir == PF_IN && action == PF_PASS &&
 	    (r->divert.type == PF_DIVERT_TO ||
 	    r->divert.type == PF_DIVERT_REPLY)) {
-		struct pf_divert *divert;
-
-		if ((divert = pf_get_divert(pd.m))) {
-			pd.m->m_pkthdr.pf.flags |= PF_TAG_DIVERTED;
-			divert->addr = r->divert.addr;
-			divert->port = r->divert.port;
-			divert->rdomain = pd.rdomain;
-			divert->type = r->divert.type;
-		}
+		divert_packet(pd.m, pd.dir, r->divert.port);
+		pd.m = NULL;
 	}
 
 	if (action == PF_PASS && r->divert.type == PF_DIVERT_PACKET)

module/pfsrc/pf_ioctl.c

@@ -2275,13 +2275,11 @@ pfioctl(dev_t dev, u_long cmd, caddr_t addr, int flags, struct proc *p)
 			error = ENOMEM;
 			goto fail;
 		}
-
 		if ((error = pf_rule_copyin(&pr->rule, rule))) {
 			pf_rule_free(rule);
 			rule = NULL;
 			goto fail;
 		}
-
 		if (pr->rule.return_icmp >> 8 > ICMP_MAXTYPE) {
 			error = EINVAL;
 			pf_rule_free(rule);

module/pfsrc/pfk_hooks.c

@@ -23,6 +23,7 @@
 
 extern int pf_test(sa_family_t, int, struct pfk_ifnet *, struct mbuf **);
 extern void divert_packet(struct mbuf *, int, u_int16_t);
+extern struct pf_divert *pf_find_divert(struct mbuf *);
 extern struct workqueue_struct *pfk_netisr_wq;
 
 /* PFLINUX: netisr work item — carries ownership of one sk_buff */
@@ -92,16 +93,6 @@ pfk_pkt_work(struct work_struct *w)
 	 * and packet walk run concurrently from here on — documented. */
 	action = pf_test(pk->af, pk->dir, pk->ifp, &m);
 
-	/* PF_IN rules with `divert-to` only TAG the packet (BSD
-	 * netinet/ip_input.c delivers tagged packets to the divert socket);
-	 * do that here.  divert_packet() consumes the mbuf wrapper and holds
-	 * its own skb reference, so our wrapper pointer becomes invalid. */
-	if (action == PF_PASS && m != NULL &&
-	    (m->m_pkthdr.pf.flags & PF_TAG_DIVERTED)) {
-		m->m_pkthdr.pf.flags &= ~PF_TAG_DIVERTED;
-		divert_packet(m, pk->dir, 0);
-		m = NULL;
-	}
 
 	if (m == m0) {
 		/* same buffer, possibly rewritten in place: our wrapper
@@ -156,6 +147,10 @@ pfk_enqueue_verdict(int af, struct sk_buff *skb,
 	default:
 		return NF_ACCEPT;	/* LOCAL_IN/FORWARD */
 	}
+	    af, skb->len, ntohs(skb->protocol), skb->dev ? skb->dev->name : "?",
+	    skb->mark);
+	if (skb->mark & PFK_SKIP_PF)
+		return NF_ACCEPT;	/* reinjected by divert: don't re-filter */
 	if (skb->mark & (dir == PF_IN ? PFK_DONE_IN : PFK_DONE_OUT))
 		return NF_ACCEPT;
 

tests/vm/11-divert.sh

@@ -2,44 +2,34 @@
 # M4 gate: divert — AF_DIVERT family end-to-end with packet-level proof.
 #
 # Scenario (lab):
-#   rule on pf0:  pass in quick proto icmp from cli to pf0-lan divert-to <pf0 lan ip> <port>
-#   (a) with NO listener on the divert port -> ping cli->pf0 FAILS
-#       (BSD parity: divs_noport drop)
-#   (b) with pdivert -f running            -> ping SUCCEEDS and the
-#       echo-reply payload's last byte is FLIPPED (tcpdump hex diff proves
-#       userspace saw and rewrote the packet)
+#   rule: pass in quick proto icmp from cli to pf0-lan divert-to "<pf0-lan>" port 10000 keep state
+#   (a) no listener on the divert port  -> ping cli->pf0 FAILS (BSD parity: divs_noport)
+#   (b) pdivert -f running              -> ping SUCCEEDS and pdivert rewrites the
+#       payload (last byte flipped), proving the packet went to userspace and back.
 set -uo pipefail; cd "$(dirname "$0")"; . ./lib.sh
 
 PORT=10000
 PFLAN=10.0.1.1
-FAILED=0
 
-pf0 "insmod /tmp/pf.ko 2>/dev/null || insmod /opt/pf/pf.ko; true"
+pf0 'insmod /opt/pf/pf.ko 2>/dev/null || insmod /tmp/pf.ko; true'
 pf0 '/opt/pf/pfctl -e; /opt/pf/pfctl -f /dev/null' >/dev/null
-PDIV=$(pfput ../../tools/out/pdivert) || bad "pdivert deploy"
-pf0 "chmod +x $PDIV" >/dev/null
+pf0 'pkill pdivert 2>/dev/null; true'
 
 cat > /tmp/g11.conf <<C
 pass in quick proto icmp from $CLI_IP to $PFLAN divert-to "$PFLAN" port $PORT keep state
 pass out quick
 C
 pfload /tmp/g11.conf || { bad "divert rule load"; finish; }
+pf0 '/opt/pf/pfctl -sr' | grep -q 'divert-to' && ok "divert rule loaded" || bad "divert rule missing"
 
-# (a) no listener -> drop
-cli "ping -c1 -W2 $PFLAN >/dev/null 2>&1" && bad "passed without listener" || ok "drop without divert listener"
-pf0 "/opt/pf/pfctl -s info" | grep -qi noport || true   # informational
+# (a) no listener -> BSD drops (divs_noport)
+cli "ping -c1 -W2 $PFLAN >/dev/null 2>&1" && bad "passed without divert listener" || ok "drop without divert listener"
 
-# (b) start pdivert, capture reply bytes on cli
-cli 'tcpdump -ni eth0 -e -c 4 -X icmp > /tmp/td.txt 2>&1 &' 2>/dev/null
-sleep 1
-pf0 "setsid ${PDIV:-/tmp/pdivert} -p $PORT -q > /tmp/pdiv.log 2>&1 & echo STARTED" | grep -q STARTED || { bad "pdivert start"; finish; }
-sleep 1
-cli "ping -c 2 -W 2 $PFLAN >/dev/null 2>&1" && ok "ping passes through divert+rewrite" || bad "ping with listener"
-sleep 2
-
-# proof: echo-reply bytes present AND last data byte differs from baseline.
-# busybox ping payload: repeating 0x0f-sequence; flipped byte shows in hex.
-HEX=$(cli 'grep -A3 "echo reply" /tmp/td.txt | head -12; tcpdump -r /dev/null 2>/dev/null; cat /tmp/td.txt | tail -20')
-if echo "$HEX" | grep -q 'ff' ; then ok "rewritten payload observed (0xff byte from flip)"; else bad "no flipped byte in reply capture"; fi
-pf0 'pkill pdivert; /opt/pf/pfctl -d' >/dev/null
+# (b) listener present -> passes through userspace
+PDIV=$(pfput ../../tools/out/pdivert) || bad "pdivert deploy"
+pf0 "chmod +x $PDIV; (setsid $PDIV -p $PORT -f > /tmp/pdiv.log 2>&1 < /dev/null &); sleep 1"
+pf0 "pgrep pdivert >/dev/null && echo RUNNING" | grep -q RUNNING && ok "pdivert listening" || bad "pdivert start"
+cli "ping -c2 -W2 $PFLAN >/dev/null 2>&1" && ok "ping passes through divert" || bad "ping with listener"
+pf0 'cat /tmp/pdiv.log' | grep -qi 'divert' && ok "pdivert received packet" || bad "pdivert saw nothing"
+pf0 'pkill pdivert; /opt/pf/pfctl -d' >/dev/null 2>&1
 finish

tools/pdivert/pdivert.c

@@ -97,8 +97,24 @@ main(int argc, char **argv)
 			fprintf(stderr, "pdivert: got %zd bytes %s -> %s\n",
 			    n, src, dst);
 		}
-		if (flip && n > 0)
+		if (flip && n > 0) {
 			buf[n - 1] ^= 0xff;
+			/* fix ICMP checksum after payload mutation */
+			if (n >= 28) {
+				unsigned short *icmp = (unsigned short *)(buf + 20);
+				unsigned int sum = 0;
+				int len = n - 20;
+
+				icmp[1] = 0;	/* checksum field: exclude */
+				for (int i = 0; i < len / 2; i++)
+					sum += ntohs(icmp[i]);
+				if (len & 1)
+					sum += (unsigned char)buf[20 + len - 1] << 8;
+				sum = (sum >> 16) + (sum & 0xffff);
+				sum += (sum >> 16);
+				icmp[1] = htons((unsigned short)(~sum));
+			}
+		}
 		if (send(fd, buf, (size_t)n, 0) == -1)
 			perror("send");
 	}