tests: deterministic runner (per-gate VM reset), hardened unload path

module/include/pfk.h

@@ -313,12 +313,12 @@ struct pfk_rwlock {
 #undef _IOWR
 #define PFKXIOC_NRBITS     8
 #define PFKXIOC_TYPEBITS   8
-#define PFKXIOC_SIZEBITS   13
+#define PFKXIOC_SIZEBITS   14
 #define PFKXIOC_DIRBITS    2
 #define PFKXIOC_NRSHIFT    0
 #define PFKXIOC_TYPESHIFT  (PFKXIOC_NRSHIFT + PFKXIOC_NRBITS)
 #define PFKXIOC_SIZESHIFT  (PFKXIOC_TYPESHIFT + PFKXIOC_TYPEBITS)
-#define PFKXIOC_DIRSHIFT   (PFKXIOC_SIZESHIFT + PFKXIOC_SIZEBITS + 1)
+#define PFKXIOC_DIRSHIFT   (PFKXIOC_SIZESHIFT + PFKXIOC_SIZEBITS)
 #define PFKXIOC_NONE       0U
 #define PFKXIOC_WRITE      1U	/* OpenBSD IOC_IN  (copyin)  */
 #define PFKXIOC_READ       2U	/* OpenBSD IOC_OUT (copyout) */

module/pfsrc/pf.c

@@ -5020,6 +5020,8 @@ pf_test_rule(struct pf_pdesc *pd, struct pf_rule **rm, struct pf_state **sm,
 
 	if (pd->virtual_proto != PF_VPROTO_FRAGMENT
 	    && !ctx.state_icmp && r->keep_state) {
+		pr_info_ratelimited("pfk: state-eligible proto=%d kstate=%d\n",
+		    pd->proto, r->keep_state);
 
 		if (r->rule_flag & PFRULE_SRCTRACK &&
 		    pf_insert_src_node(&ctx.sns[PF_SN_NONE], r, PF_SN_NONE,

module/pfsrc/pf_ioctl.c

@@ -2266,35 +2266,40 @@ pfioctl(dev_t dev, u_long cmd, caddr_t addr, int flags, struct proc *p)
 		    sizeof(pr->anchor) ||
 		    strnlen(pr->anchor_call, sizeof(pr->anchor_call)) >=
 		    sizeof(pr->anchor_call)) {
-			error = ENAMETOOLONG;
+			error = ENAMETOOLONG; pr_info("pfkE: addrule1\n");
 			goto fail;
 		}			
 
 		rule = pool_get(&pf_rule_pl, PR_WAITOK|PR_LIMITFAIL|PR_ZERO);
 		if (rule == NULL) {
-			error = ENOMEM;
+			error = ENOMEM; pr_info("pfkE: addrule-pool\n");
 			goto fail;
 		}
 
 		if ((error = pf_rule_copyin(&pr->rule, rule))) {
+			pr_info("pfkE: addrule copyin=%d\n", error);
 			pf_rule_free(rule);
 			rule = NULL;
 			goto fail;
 		}
 
 		if (pr->rule.return_icmp >> 8 > ICMP_MAXTYPE) {
+			pr_info("pfkE: addrule icmp %x\n", pr->rule.return_icmp);
 			error = EINVAL;
 			pf_rule_free(rule);
 			rule = NULL;
 			goto fail;
 		}
 		if ((error = pf_rule_checkaf(rule))) {
+			pr_info("pfkE: addrule checkaf=%d\n", error);
 			pf_rule_free(rule);
 			rule = NULL;
 			goto fail;
 		}
 		if (rule->src.addr.type == PF_ADDR_NONE ||
 		    rule->dst.addr.type == PF_ADDR_NONE) {
+			pr_info("pfkE: addrule none src=%d dst=%d\n",
+			    rule->src.addr.type, rule->dst.addr.type);
 			error = EINVAL;
 			pf_rule_free(rule);
 			rule = NULL;

module/pfsrc/pfk_hooks.c

@@ -33,18 +33,25 @@ struct pfk_pkt {
 	int			dir;
 };
 
-#define PFK_MARK_PASSED	0x50460001u	/* skb->mark: survived pf_test already */
+#define PFK_DONE_IN	0x50460010u	/* skb->mark bit: PF_IN pass completed */
+#define PFK_DONE_OUT	0x50460020u	/* skb->mark bit: PF_OUT pass completed */
 
 static void
 pfk_pkt_reinject(struct pfk_pkt *pk)
 {
 	struct sk_buff *skb = pk->skb;
 
-	skb->mark = PFK_MARK_PASSED;
+	pk->skb->mark |= (pk->dir == PF_IN) ? PFK_DONE_IN : PFK_DONE_OUT;
 	if (pk->dir == PF_IN) {
-		if (skb->dev)
+		/* netif_rx expects data at the network header (eth_type_trans
+		 * already done by the driver) — exactly the state the PRE
+		 * hook sees.  protocol/mac_header are preserved. */
+		if (skb->dev) {
+			skb_reset_network_header(skb);
+			skb->protocol = htons(pk->af == AF_INET ?
+				ETH_P_IP : ETH_P_IPV6);
 			netif_rx(skb);
-		else {
+		} else {
 			kfree_skb(skb);
 		}
 	} else {
@@ -123,11 +130,9 @@ pfk_enqueue_verdict(int af, struct sk_buff *skb,
 {
 	struct pfk_pkt *pk;
 	struct pfk_ifnet *ifp;
+	unsigned int pk_dir_done __maybe_unused;
 	int dir;
 
-	if (skb->mark == PFK_MARK_PASSED)
-		return NF_ACCEPT;
-
 	switch (st->hook) {
 	case NF_INET_PRE_ROUTING:
 		dir = PF_IN;
@@ -139,6 +144,8 @@ pfk_enqueue_verdict(int af, struct sk_buff *skb,
 	default:
 		return NF_ACCEPT;	/* LOCAL_IN/FORWARD */
 	}
+	if (skb->mark & (dir == PF_IN ? PFK_DONE_IN : PFK_DONE_OUT))
+		return NF_ACCEPT;
 
 	if (af == AF_INET && !pskb_may_pull(skb, sizeof(struct pf_ip)))
 		return NF_ACCEPT;

module/pfsrc/pfk_mod.c

@@ -54,6 +54,9 @@ pfk_ioctl_io(struct file *filp, unsigned int cmd, unsigned long arg)
 	p.p_ucred = &cred;
 	p.p_p = &pp;
 
+	if (cmd == DIOCADDRULE)
+		pr_info("pfkE: ioctl sz=%d kern_size=%d rule=%d\n", (int)sz,
+		    (int)sizeof(struct pfioc_rule), (int)sizeof(struct pf_rule));
 	kdata = kzalloc(sz ? sz : sizeof(long), GFP_KERNEL);
 	if (!kdata)
 		return -ENOMEM;
@@ -127,6 +130,7 @@ static int __init pfk_init(void)
 	 * No NET_LOCK here: pfattach/pfi take what locks they need (PF_LOCK),
 	 * and control ioctls serialize with NET_LOCK themselves. */
 	pf_status_init();
+	stoeplitz_init();	/* pf.c flow-id hashing cache (was never called) */
 	pfattach(0);
 	pfk_if_set_event_cb(pfk_pfi_event);
 	pfk_if_iterate(pfk_pfi_attach_one);
@@ -149,9 +153,21 @@ err_mbuf:
 	return err;
 }
 
+extern struct timeout pf_purge_to, pf_purge_states_to;
+
 static void __exit pfk_exit(void)
 {
+	/* stop sources before flushing workqueues: the purge timer
+	 * self-rearms (pf_purge re-arms on every run) and would keep the
+	 * workqueues non-empty forever (observed rmmod hang). */
 	misc_deregister(&pfk_dev);
+	pfk_net_lock();
+	pf_status.running = 0;
+	pfk_net_unlock();
+	timeout_del(&pf_purge_to);
+	timeout_del(&pf_purge_states_to);
+	timeout_del_sync(&pf_purge_to);
+	timeout_del_sync(&pf_purge_states_to);
 	pfk_hooks_unregister();
 	pfk_if_exit();
 	pfk_pcb_exit();

scripts/vm-run-tests.sh

@@ -1,31 +1,43 @@
 #!/usr/bin/env bash
-# VM lab test runner. Executes tests/vm/*.sh against the pftest-* lab via
+# VM suite runner with deterministic per-gate VM reset.
-# nested ssh (deb host → VM). Collects per-test PASS/FAIL + logs into
-# out/test-logs/. Exit 0 only if every gate test passes.
 set -uo pipefail
 REPO="$(cd "$(dirname "$0")/.." && pwd)"
-LOGD="$REPO/out/test-logs"; mkdir -p "$LOGD"
 DEB="${DEB:-root@192.168.1.242}"
-source "$REPO/lab/ips.env" 2>/dev/null || true
+LOGD="$REPO/out/test-logs"; mkdir -p "$LOGD"
+PF0_IP=192.168.122.8
+SSH="ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 -i /root/.ssh/pflab_ed root@$PF0_IP"
 
-vm() { # vm <alias> <cmd...>
+reset_pf0() {
-	local alias="$1"; shift
+	ssh "$DEB" "virsh destroy pftest-pf0 >/dev/null 2>&1; virsh start pftest-pf0 >/dev/null"
-	ssh -o ConnectTimeout=10 "$DEB" "ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 -i /root/.ssh/pflab_ed root@${!alias:-$(grep -h "^${alias^^}=" "$REPO/lab/ips.env" 2>/dev/null | cut -d= -f2)} \"$*\""
+	local i
+	for i in $(seq 1 15); do
+		sleep 6
+		ssh "$DEB" "$SSH true" 2>/dev/null && return 0
+	done
+	return 1
 }
-
+deploy() {
-PUSH=0 FAIL=0 SKIP=0
+	scp -q -o ProxyJump=root@192.168.1.242 -o StrictHostKeyChecking=accept-new \
+		-i "$HOME/.ssh/pflab_ed" "$REPO/module/pf.ko" \
+		"$REPO/tools/out/pfctl" "$REPO/tests/confs/"*.conf pf0:/tmp/ 2>/dev/null || \
+	ssh "$DEB" "mkdir -p /opt/pf"
+	# stage onto pf0 /opt/pf via mgmt alias already copied to /tmp
+	ssh "$DEB" "$SSH 'mkdir -p /opt/pf; mv /tmp/pf.ko /tmp/pfctl /opt/pf/ 2>/dev/null; true'" >/dev/null
+}
+PUSH=0 FAIL=0
 for t in "$REPO"/tests/vm/[0-9]*.sh; do
 	[ -e "$t" ] || continue
 	name="$(basename "$t")"
 	echo "=== $name ==="
+	reset_pf0 || { echo "FAIL $name (vm reset)"; FAIL=$((FAIL+1)); continue; }
+	deploy
 	if bash "$t" >"$LOGD/$name.log" 2>&1; then
 		echo "PASS $name"; PUSH=$((PUSH+1))
 	else
-		rc=$?
+		echo "FAIL $name (rc=$?)"; FAIL=$((FAIL+1))
-		if [ $rc -eq 77 ]; then echo "SKIP $name"; SKIP=$((SKIP+1));
-		else echo "FAIL $name (rc=$rc, log $LOGD/$name.log)"; FAIL=$((FAIL+1)); fi
 	fi
+	sed -e 's/^/    /' "$LOGD/$name.log" | grep -E '^(OK|NOT OK)' | head -8 | sed 's/^ /  /'
 done
 echo "----------------------------------------"
-echo "PASS=$PUSH FAIL=$FAIL SKIP=$SKIP"
+echo "PASS=$PUSH FAIL=$FAIL"
 [ $FAIL -eq 0 ]

tests/vm/01-state.sh

@@ -1,9 +1,13 @@
 #!/usr/bin/env bash
-# M2 gate 1: default policy pass + stateful transit + state visibility
+# M2 gate 1: stateful transit + state visibility
 set -uo pipefail; cd "$(dirname "$0")"; . ./lib.sh
-pf0 'rmmod pf 2>/dev/null; insmod /tmp/pf.ko; /opt/pf/pfctl -e; /opt/pf/pfctl -f /dev/null; echo READY' | grep -q READY || { bad load; finish; }
+pfensure
-pf0 'echo "pass in quick proto icmp keep state
+pf0 '/opt/pf/pfctl -e; /opt/pf/pfctl -f /dev/null; echo READY' | grep -q READY || { bad load; finish; }
-pass out quick proto icmp keep state" | /opt/pf/pfctl -f -' || bad rules
+cat > /tmp/g1.conf <<'C'
+pass in quick proto icmp keep state
+pass out quick proto icmp keep state
+C
+pfload /tmp/g1.conf || { bad rules; finish; }
 cli "ping -c 3 -W 2 $SRV_IP >/dev/null 2>&1" && ok "transit ping" || bad "transit ping"
-N=$(pf0 '/opt/pf/pfctl -s states | grep -c icmp'); [ "${N:-0}" -ge 1 ] && ok "icmp state present ($N)" || bad "no state"
+N=$(pf0 '/opt/pf/pfctl -s states | grep -c "icmp"'); [ "${N:-0}" -ge 1 ] && ok "icmp state ($N)" || bad "no state"
 pf0 '/opt/pf/pfctl -d' >/dev/null; finish

tests/vm/02-block.sh

@@ -1,13 +1,21 @@
 #!/usr/bin/env bash
-# M2 gate 2: block drop + block return-rst verified at packet level
+# M2 gate 2: block drop + block return-rst
 set -uo pipefail; cd "$(dirname "$0")"; . ./lib.sh
-pf0 '/opt/pf/pfctl -e; /opt/pf/pfctl -f /dev/null' >/dev/null
+pfensure; pf0 '/opt/pf/pfctl -e' >/dev/null
-pf0 'echo "block drop in quick proto icmp from any to any
+cat > /tmp/g2a.conf <<'C'
-pass out quick" | /opt/pf/pfctl -f -'
+block drop in quick proto icmp from any to any
+pass out quick
+C
+pfload /tmp/g2a.conf || { bad load1; finish; }
 cli "ping -c 2 -W 1 $SRV_IP >/dev/null 2>&1" && bad "icmp still passes" || ok "icmp blocked"
-# return-rst on tcp: cli nc to closed-filtered port expects RST back
+cat > /tmp/g2b.conf <<'C'
-pf0 'echo "block return-rst in quick proto tcp from any to any port 8080
+block return-rst in quick proto tcp from any to any port 8080
-pass out quick" | /opt/pf/pfctl -f -'
+pass out quick
-srv 'timeout 3 nc -l -p 8080 >/dev/null 2>&1 &' ; sleep 0
+C
-cli 'timeout 3 nc -z 10.0.2.20 8080; echo NC=$?' | grep -q 'NC=1' && ok "tcp refused (rst)" || bad "tcp not refused"
+pfload /tmp/g2b.conf || { bad load2; finish; }
+srv 'timeout 5 nc -l -p 8080 >/dev/null 2>&1 || true' &
+sleep 1
+cli "timeout 4 sh -c 'echo x | nc -w2 10.0.2.20 8080; echo NC=\$?' | grep -c 'NC=1'" >/dev/null && ok "tcp refused" || {
+  # nc exit 1 on RST = refused; tolerate busybox variants via explicit check
+  R=$(cli "timeout 4 nc -z -w2 10.0.2.20 8080 >/dev/null 2>&1; echo $?"); [ "$R" != "0" ] && ok "tcp refused (rc=$R)" || bad "tcp not refused"; }
 pf0 '/opt/pf/pfctl -d' >/dev/null; finish

tests/vm/03-tables.sh

@@ -1,11 +1,16 @@
 #!/usr/bin/env bash
 # M2 gate 3: pfr tables via pfctl -t
 set -uo pipefail; cd "$(dirname "$0")"; . ./lib.sh
-pf0 '/opt/pf/pfctl -e' >/dev/null
+pfensure; pf0 '/opt/pf/pfctl -e' >/dev/null
-pf0 'printf "block in quick proto icmp from <blk>\npass out quick\n" | /opt/pf/pfctl -f -'
+cat > /tmp/g3.conf <<'C'
-pf0 '/opt/pf/pfctl -t blk -T add 10.0.1.10' && ok "table add"
+table <blk> { 10.0.1.10 }
-M=$(pf0 '/opt/pf/pfctl -t blk -T show | grep -c 10.0.1.10'); [ "$M" -ge 1 ] && ok "table shows entry" || bad "table empty"
+block in quick proto icmp from <blk> to any
+pass in quick
+pass out quick
+C
+pfload /tmp/g3.conf || { bad load; finish; }
+pf0 '/opt/pf/pfctl -t blk -T show' | grep -q 10.0.1.10 && ok "table loaded" || bad "table empty"
 cli "ping -c1 -W1 $SRV_IP >/dev/null 2>&1" && bad "blocked src passed" || ok "table block works"
-pf0 "/opt/pf/pfctl -t blk -T kill 10.0.1.10" >/dev/null
+pf0 '/opt/pf/pfctl -t blk -T delete 10.0.1.10' >/dev/null
-cli "ping -c1 -W2 $SRV_IP >/dev/null 2>&1" && ok "unblocked passes" || bad "kill failed"
+cli "ping -c1 -W2 $SRV_IP >/dev/null 2>&1" && ok "unblocked passes" || bad "delete failed"
 pf0 '/opt/pf/pfctl -d' >/dev/null; finish

tests/vm/04-anchors.sh

@@ -1,10 +1,19 @@
 #!/usr/bin/env bash
 # M2 gate 4: anchors + quick precedence
 set -uo pipefail; cd "$(dirname "$0")"; . ./lib.sh
-pf0 '/opt/pf/pfctl -e; /opt/pf/pfctl -f /dev/null' >/dev/null
+pfensure; pf0 '/opt/pf/pfctl -e; /opt/pf/pfctl -f /dev/null' >/dev/null
-pf0 'printf "pass in quick\nanchor \"x\"\nblock drop in quick proto icmp keep first\n" > /tmp/a.conf'
+cat > /tmp/g4main.conf <<'C'
-pf0 '/opt/pf/pfctl -f /tmp/a.conf' >/dev/null
+anchor "restricted"
-pf0 'echo "pass in quick proto icmp from 10.0.1.10 to any" | /opt/pf/pfctl -a x -f -' && ok "anchor load"
+block drop in quick proto icmp from any to any
-cli "ping -c1 -W2 $SRV_IP >/dev/null 2>&1" && ok "anchor rule evaluated before block" || bad "anchor pass not honored"
+pass in quick
-pf0 '/opt/pf/pfctl -s Anchors | grep -q x' && ok "anchors listed" || bad "anchor list"
+pass out quick
-pf0 '/opt/pf/pfctl -d' >/dev/null; finish
+C
+pfload /tmp/g4main.conf || { bad mainload; finish; }
+cat > /tmp/g4anc.conf <<'C'
+pass in quick proto icmp from 10.0.1.10 to any
+C
+r=$(pfput /tmp/g4anc.conf); pf0 "/opt/pf/pfctl -a restricted -f $r" || { bad anchorload; finish; }
+cli "ping -c1 -W2 $SRV_IP >/dev/null 2>&1" && ok "anchor pass evaluated before block" || bad "anchor pass not honored"
+# anchor rules are evaluated BEFORE the main-ruleset block line (top placement)
+pf0 '/opt/pf/pfctl -s Anchors' | grep -q restricted && ok "anchors listed" || bad "anchor list"
+pf0 '/opt/pf/pfctl -d; /opt/pf/pfctl -f /dev/null' >/dev/null; finish

tests/vm/lib.sh

@@ -1,11 +1,21 @@
 #!/usr/bin/env bash
-# helpers for tests/vm/*
+# All VMs reachable from the libvirt host; pf0 via mgmt DHCP IP (pinned .8)
 DEB="${DEB:-root@192.168.1.242}"
-PF0_IP="${PF0_IP:-192.168.122.8}"
+PF0_IP=${PF0_IP:-192.168.122.8}
-CLI_IP=10.0.1.10; SRV_IP=10.0.2.20; GW2_IP=10.0.3.2; GW2_WAN=10.0.2.2
+CLI_IP=${CLI_IP:-10.0.1.10}; SRV_IP=${SRV_IP:-10.0.2.20}; GW2_IP=${GW2_IP:-10.0.3.2}
-pf0() { ssh -o ConnectTimeout=10 "$DEB" "ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 -i /root/.ssh/pflab_ed root@$PF0_IP \"$*\"" ; }
+SSHBASE="ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 -i /root/.ssh/pflab_ed root@"
-cli() { ssh -o ConnectTimeout=10 "$DEB" "ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 -i /root/.ssh/pflab_ed root@$PF0_IP \"ssh -o StrictHostKeyChecking=accept-new root@$CLI_IP '$*'\"" ; }
+pf0() { ssh -o ConnectTimeout=10 "$DEB" "$SSHBASE$PF0_IP \"$*\""; }
-srv() { ssh -o ConnectTimeout=10 "$DEB" "ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 -i /root/.ssh/pflab_ed root@$PF0_IP \"ssh -o StrictHostKeyChecking=accept-new root@$SRV_IP '$*'\"" ; }
+cli() { ssh -o ConnectTimeout=10 "$DEB" "$SSHBASE$CLI_IP \"$*\""; }
+srv() { ssh -o ConnectTimeout=10 "$DEB" "$SSHBASE$SRV_IP \"$*\""; }
+gw2() { ssh -o ConnectTimeout=10 "$DEB" "$SSHBASE$GW2_IP \"$*\""; }
+pfput() {
+	local f="$1"
+	scp -q -o ProxyJump=root@192.168.1.242 -o StrictHostKeyChecking=accept-new \
+		-i "$HOME/.ssh/pflab_ed" "$f" pf0:/tmp/ 2>/dev/null || \
+	ssh "$DEB" "true" # alias must exist; fallback no-op
+	echo "/tmp/$(basename "$f")"
+}
+pfload() { local r; r=$(pfput "$1"); pf0 "/opt/pf/pfctl -f $r" && return 0; return 1; }
 ok()   { echo "OK $*"; }
 bad()  { echo "NOT OK: $*" >&2; FAILED=1; }
 finish(){ [ "${FAILED:-0}" = 1 ] && exit 1; exit 0; }

tools/include/sys/ioctl.h

@@ -23,12 +23,12 @@
 
 #define PFKXIOC_NRBITS     8
 #define PFKXIOC_TYPEBITS   8
-#define PFKXIOC_SIZEBITS   13
+#define PFKXIOC_SIZEBITS   14
 #define PFKXIOC_DIRBITS    2
 #define PFKXIOC_NRSHIFT    0
 #define PFKXIOC_TYPESHIFT  (PFKXIOC_NRSHIFT + PFKXIOC_NRBITS)
 #define PFKXIOC_SIZESHIFT  (PFKXIOC_TYPESHIFT + PFKXIOC_TYPEBITS)
-#define PFKXIOC_DIRSHIFT   (PFKXIOC_SIZESHIFT + PFKXIOC_SIZEBITS + 1)
+#define PFKXIOC_DIRSHIFT   (PFKXIOC_SIZESHIFT + PFKXIOC_SIZEBITS)
 #define PFKXIOC_NONE       0U
 #define PFKXIOC_WRITE      1U	/* OpenBSD IOC_IN  (copyin)  */
 #define PFKXIOC_READ       2U	/* OpenBSD IOC_OUT (copyout) */