M2: live netfilter data path (pf_test per direction), pfattach bring-up, NET_LOCK reentrancy fix, static pfctl

- pfk_hooks: PRE→PF_IN, LOCAL_OUT/POST→PF_OUT v4+v6; replacement/buffer
  handling with NF_STOLEN + re-injection (documented divergence: re-entry)
- pfk_mod: pf_status_init/pfattach + pfi attach via pfk_if event callback
- pfk_kern: NET_LOCK holder reentrancy (fix deadlock when notifier fires
  under the write lock — observed as full-VM network hang)
- tools: static pfctl link for musl VMs

module/compat/pfk_if.c

@@ -7,6 +7,8 @@
  */
 #include <pfk.h>
 
+static void pfk_if_event_call(int added, struct pfk_ifnet *ifp);
+
 static DEFINE_IDR(pfk_ifnet_idr);
 static DEFINE_SPINLOCK(pfk_ifnet_lock);
 static struct pfk_ifnet *pfk_egress;
@@ -290,6 +292,7 @@ pfk_netdev_event(struct notifier_block *nb, unsigned long ev, void *ptr)
 			return NOTIFY_DONE;
 		}
 		spin_unlock_irqrestore(&pfk_ifnet_lock, flags);
+		pfk_if_event_call(1, ifp);
 		break;
 	case NETDEV_UNREGISTER:
 		spin_lock_irqsave(&pfk_ifnet_lock, flags);
@@ -297,6 +300,7 @@ pfk_netdev_event(struct notifier_block *nb, unsigned long ev, void *ptr)
 		if (ifp) {
 			ifp->alive = 0;
 			idr_remove(&pfk_ifnet_idr, dev->ifindex);
+			pfk_if_event_call(0, ifp);
 			call_rcu(&ifp->rcu, pfk_ifnet_release);
 		}
 		spin_unlock_irqrestore(&pfk_ifnet_lock, flags);
@@ -325,6 +329,35 @@ static struct notifier_block pfk_netdev_nb = {
 	.notifier_call = pfk_netdev_event,
 };
 
+static void (*pfk_if_event_cb)(int added, struct pfk_ifnet *ifp);
+
+static void
+pfk_if_event_call(int added, struct pfk_ifnet *ifp)
+{
+	if (!pfk_if_event_cb)
+		return;
+	/* netdev notifiers run in process context; NET lock is sleepable */
+	pfk_net_lock();
+	pfk_if_event_cb(added, ifp);
+	pfk_net_unlock();
+}
+
+void
+pfk_if_set_event_cb(void (*cb)(int added, struct pfk_ifnet *ifp))
+{
+	pfk_if_event_cb = cb;
+}
+
+void
+pfk_if_iterate(void (*cb)(struct pfk_ifnet *ifp))
+{
+	struct pfk_ifnet *ifp;
+	int id = -1;
+
+	while ((ifp = pfk_if_next_from(&id)) != NULL)
+		cb(ifp);
+}
+
 /* -------- egress -------- */
 void
 pfk_egress_refresh(void)
@@ -403,8 +436,10 @@ pfk_if_init(void)
 		else
 			ifp = NULL;	/* already tracked by notifier */
 		spin_unlock_irqrestore(&pfk_ifnet_lock, flags);
-		if (ifp)
+		if (ifp) {
 			pfk_if_refresh(ifp);
+			pfk_if_event_call(1, ifp);
+		}
 	}
 	rtnl_unlock();
 	pfk_egress_refresh();

module/compat/pfk_kern.c

@@ -100,10 +100,28 @@ taskq_create(const char *name, unsigned int nthreads, int ipl,
 /* ---------------- NET_LOCK ---------------- */
 static DECLARE_RWSEM(pfk_net_sem);
 static atomic_t pfk_net_readers = ATOMIC_INIT(0);
+static atomic_t pfk_net_owner = ATOMIC_INIT(0);	/* task pid holding write */
 
 void pfk_net_lock_init(void) { }
-void pfk_net_lock(void)        { down_write(&pfk_net_sem); }
-void pfk_net_unlock(void)      { up_write(&pfk_net_sem); }
+void
+pfk_net_lock(void)
+{
+	/* Re-entrant for the holder: OpenBSD's NET_LOCK is friendly with
+	 * nested calls in the same path (e.g. our netdev notifier firing
+	 * while pfioctl holds the lock). */
+	if (atomic_read(&pfk_net_owner) == current->pid)
+		return;
+	down_write(&pfk_net_sem);
+	atomic_set(&pfk_net_owner, current->pid);
+}
+void
+pfk_net_unlock(void)
+{
+	if (atomic_read(&pfk_net_owner) != current->pid)
+		return;		/* nested no-op for non-holders */
+	atomic_set(&pfk_net_owner, 0);
+	up_write(&pfk_net_sem);
+}
 void pfk_net_read_enter(void)  { down_read(&pfk_net_sem);
 				 atomic_inc(&pfk_net_readers); }
 void pfk_net_read_release(void){ atomic_dec(&pfk_net_readers);

module/compat/pfk_mbuf.c

@@ -50,6 +50,30 @@ pfk_m_setpkthdr(struct mbuf *m)
 	m->m_pkthdr.tags = NULL;
 }
 
+/*
+ * Ingress wrapper for netfilter hook skbs: takes an EXTRA skb reference so
+ * the wrapper can be consumed (m_freem) without disturbing netfilter's own
+ * reference; fills the pkthdr the way OpenBSD's ip_input/ip_output does.
+ */
+struct mbuf *
+pfk_m_wrap_skb(struct sk_buff *skb, int af)
+{
+	struct mbuf *m;
+
+	skb_get(skb);
+	m = m_alloc_skb(skb, M_NOWAIT);
+	if (!m)
+		return NULL;
+	m->m_pkthdr.af = af;
+	m->m_pkthdr.proto = (af == AF_INET) ?
+		((struct pf_ip *)skb->data)->ip_p : -1;
+	m->m_pkthdr.ph_ifidx = skb->dev ? skb->dev->ifindex : 0;
+	m->m_pkthdr.ifp = m->m_pkthdr.rcvif =
+		skb->dev ? pfk_ifp_bydev(skb->dev) : NULL;
+	m->m_pkthdr.ph_rtableid = 0;
+	return m;
+}
+
 struct mbuf *
 m_alloc_skb(struct sk_buff *skb, int how)
 {

module/include/pfk_if.h

@@ -86,6 +86,12 @@ struct pfk_ifnet *pfk_if_next(struct pfk_ifnet *ifp);
 struct pfk_ifnet *pfk_if_next_from(int *idp);
 struct net_device *ip_rt_get_default_dev(struct net *net);
 
+/* Notification for PF engine (pfi attach/detach), called under NET_LOCK:
+ * added=1 new interface (after populate), added=0 interface going away. */
+void pfk_if_set_event_cb(void (*cb)(int added, struct pfk_ifnet *ifp));
+/* Iterate all live interfaces with cb(ifp) under NET_LOCK-read. */
+void pfk_if_iterate(void (*cb)(struct pfk_ifnet *ifp));
+
 /* xmit API used by pf_route/pf output (implemented in pfk_xmit.c).
  * PFLINUX: `dst` is void * so BSD accessor macros (sintosa/sin6tosa,
  * which keep the sockaddr_in/sockaddr_in6 type for field access) can be

module/include/pfk_mbuf.h

@@ -141,6 +141,8 @@ struct mbuf {
 struct mbuf *m_get(int how, short type);
 struct mbuf *m_gethdr(int how, short type);
 struct mbuf *m_alloc_skb(struct sk_buff *skb, int how); /* wrap borrowed skb */
+struct mbuf *pfk_m_wrap_skb(struct sk_buff *skb, int af); /* ingress: extra ref */
+struct mbuf *pfk_m_wrap_skb(struct sk_buff *skb, int af); /* ingress: takes extra ref */
 struct mbuf *m_devget(char *buf, int totlen, int off,
 		      struct pfk_ifnet *ifp, void (*copyin_fc)(caddr_t, int));
 void  pfk_m_setpkthdr(struct mbuf *m);

module/pfsrc/pfk_hooks.c

@@ -1,37 +1,127 @@
 /*
- * PF-on-Linux netfilter hook registration + engine status surface.
- * The verdict path is staged pf.c (pf_test()); hook→pf_test wiring with
- * mbuf ingress is milestone M2's data-path task — until then the hooks
- * accept so the module is inert once loaded (never silent: pr_info at
- * registration says so). (ISC 2026)
+ * PF-on-Linux netfilter hook registration + M2 data path.
+ *
+ * One pf_test() pass per BSD ip-stack call site:
+ *   NF_INET_PRE_ROUTING  → pf_test(PF_IN)   (OpenBSD ip_input)
+ *   NF_INET_LOCAL_OUT    → pf_test(PF_OUT)  (OpenBSD ip_output, early)
+ *   NF_INET_POST_ROUTING → pf_test(PF_OUT)  (OpenBSD ip_output, routed)
+ * LOCAL_IN / FORWARD are NF_ACCEPT: the packet was already evaluated on
+ * its PRE (input) and POST (output) passes, exactly like OpenBSD which
+ * runs PF once per direction inside ip_input/ip_output.
+ *
+ * Replacements/deferrals (fragment reassembly, divert, refragmentation)
+ * re-enter the stack via netif_rx()/pfk_ip_output() — PFLINUX divergence:
+ * OpenBSD continues in place; re-entry re-runs PRE on the survivor, which
+ * for reassembled v4 datagrams yields the same state/rule outcome (the
+ * reassembled packet no longer matches fragment-only rules).  Documented
+ * in docs/porting-notes.md.
+ *
+ * (c) 2026 PF-on-Linux contributors — ISC
  */
 #include <pfk.h>
 #include <pfk_api.h>
 #include <net/pfvar.h>
 #include <net/pfvar_priv.h>
 
-static int pfk_enabled = 1;	/* like pf_defaultwise/pf_enable on boot */
-static struct pf_status pfk_stat;
+extern int pf_test(sa_family_t, int, struct pfk_ifnet *, struct mbuf **);
 
-void pfk_status(struct pf_status *s)
-{
-	if (s)
-		*s = pfk_stat;
-}
-
-int pfk_enable(int on)
-{
-	pfk_enabled = on ? 1 : 0;
-	return 0;
-}
-
-/* --- netfilter hooks: ingress/egress plumbing --- */
+/* shared verdict plumbing for v4/v6 (af decides header pull size) */
 static unsigned int
-pfk_ipv4_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *st)
+pfk_run(int af, struct sk_buff *skb, const struct nf_hook_state *st)
 {
-	/* PFLINUX M1: pf_test() routing arrives with the M2 data-path task;
-	 * until then the hooks accept (module inert). */
-	return NF_ACCEPT;
+	struct pfk_ifnet *ifp;
+	struct mbuf *m, *m0;
+	int dir, action;
+
+	if (af == AF_INET &&
+	    !pskb_may_pull(skb, sizeof(struct pf_ip)))
+		return NF_ACCEPT;
+	if (af == AF_INET6 &&
+	    !pskb_may_pull(skb, sizeof(struct pf_ip6_hdr)))
+		return NF_ACCEPT;
+
+	switch (st->hook) {
+	case NF_INET_PRE_ROUTING:
+		dir = PF_IN;
+		break;
+	case NF_INET_LOCAL_OUT:
+	case NF_INET_POST_ROUTING:
+		dir = PF_OUT;
+		break;
+	default:
+		return NF_ACCEPT;	/* LOCAL_IN/FORWARD: seen already */
+	}
+
+	/* ifp: OpenBSD passes the *input* interface on PF_IN and the
+	 * output interface on PF_OUT (from the route). */
+	if (dir == PF_IN)
+		ifp = pfk_ifp_bydev(st->in ? st->in : skb->dev);
+	else
+		ifp = pfk_ifp_bydev(skb->dev ? skb->dev : st->out);
+	if (!ifp)
+		return NF_ACCEPT;	/* unknown iface: inert, like no pf */
+
+	m = pfk_m_wrap_skb(skb, af);
+	if (!m)
+		return NF_ACCEPT;
+	m0 = m;
+
+	pfk_net_read_enter();
+	action = pf_test(af, dir, ifp, &m);
+	pfk_net_read_release();
+
+	if (m == m0) {
+		/* in-place (possible header rewrite already reflected in
+		 * the shared skb); our extra ref is released by m_freem */
+		m_freem(m);
+		return (action == PF_PASS) ? NF_ACCEPT : NF_DROP;
+	}
+
+	/* m != m0: either NULL (consumed: buffered fragment, divert,
+	 * refragment-then-free, drop-after-replacement) or a *new* mbuf
+	 * (reassembly survivor). m0 itself was consumed by pf. */
+	if (m == NULL) {
+		kfree_skb(skb);		/* we own the hook ref now */
+		return NF_STOLEN;
+	}
+	/* survivor: drop the netfilter original, re-inject the survivor */
+	kfree_skb(skb);
+	if (dir == PF_IN) {
+		struct sk_buff *ns = m_to_skb(m);
+
+		ns->protocol = htons(af == AF_INET ? ETH_P_IP : ETH_P_IPV6);
+		if (!ns->dev)
+			ns->dev = st->in;
+		if (ns->dev)
+			dev_hold(ns->dev);
+		else {
+			kfree_skb(ns);
+			m->skb = NULL;
+			m_freem(m);
+			return NF_STOLEN;
+		}
+		netif_rx(ns);
+		m->skb = NULL;	/* reference moved into netif_rx */
+		m_freem(m);
+	} else {
+		/* survivor leaves via the BSD output helper (consumes m) */
+		pfk_ip_output(m, NULL, NULL, 0, NULL, NULL, 0);
+	}
+	return NF_STOLEN;
+}
+
+static unsigned int
+pfk_ipv4_hook(void *priv, struct sk_buff *skb,
+	const struct nf_hook_state *st)
+{
+	return pfk_run(AF_INET, skb, st);
+}
+
+static unsigned int
+pfk_ipv6_hook(void *priv, struct sk_buff *skb,
+	const struct nf_hook_state *st)
+{
+	return pfk_run(AF_INET6, skb, st);
 }
 
 static struct nf_hook_ops pfk_hooks[] = {
@@ -45,6 +135,16 @@ static struct nf_hook_ops pfk_hooks[] = {
 	  .priority = NF_IP_PRI_CONNTRACK + 1 },
 	{ .hook = pfk_ipv4_hook, .pf = NFPROTO_IPV4, .hooknum = NF_INET_POST_ROUTING,
 	  .priority = NF_IP_PRI_LAST - 1 },
+	{ .hook = pfk_ipv6_hook, .pf = NFPROTO_IPV6, .hooknum = NF_INET_PRE_ROUTING,
+	  .priority = NF_IP_PRI_FIRST },
+	{ .hook = pfk_ipv6_hook, .pf = NFPROTO_IPV6, .hooknum = NF_INET_LOCAL_IN,
+	  .priority = NF_IP_PRI_FIRST + 1 },
+	{ .hook = pfk_ipv6_hook, .pf = NFPROTO_IPV6, .hooknum = NF_INET_FORWARD,
+	  .priority = NF_IP_PRI_FIRST },
+	{ .hook = pfk_ipv6_hook, .pf = NFPROTO_IPV6, .hooknum = NF_INET_LOCAL_OUT,
+	  .priority = NF_IP_PRI_CONNTRACK + 1 },
+	{ .hook = pfk_ipv6_hook, .pf = NFPROTO_IPV6, .hooknum = NF_INET_POST_ROUTING,
+	  .priority = NF_IP_PRI_LAST - 1 },
 };
 #define PFK_NHOOKS ARRAY_SIZE(pfk_hooks)
 
@@ -53,7 +153,8 @@ pfk_hooks_register(void)
 {
 	int i, err;
 
-	pr_info("pfk: netfilter hooks registered (verdict path inert until M2)\n");
+	pr_info("pfk: netfilter hooks registering (%zu, verdict path live)\n",
+		(size_t)PFK_NHOOKS);
 	for (i = 0; i < PFK_NHOOKS; i++) {
 		pfk_hooks[i].priv = NULL;
 		err = nf_register_net_hook(&init_net, &pfk_hooks[i]);

module/pfsrc/pfk_mod.c

@@ -91,6 +91,24 @@ static struct miscdevice pfk_dev = {
 	.mode	= 0600,
 };
 
+extern void pf_status_init(void);
+extern int pfattach(int);
+
+static void
+pfk_pfi_event(int added, struct pfk_ifnet *ifp)
+{
+	if (added)
+		pfi_attach_ifnet(ifp);
+	else
+		pfi_detach_ifnet(ifp);
+}
+
+static void
+pfk_pfi_attach_one(struct pfk_ifnet *ifp)
+{
+	pfi_attach_ifnet(ifp);
+}
+
 static int __init pfk_init(void)
 {
 	int err;
@@ -105,6 +123,13 @@ static int __init pfk_init(void)
 	err = pfk_if_init();
 	if (err)
 		goto err_pcb;
+	/* OpenBSD bring-up order: status/pools/rulesets, then kifs */
+	pfk_net_lock();
+	pf_status_init();
+	pfattach(0);
+	pfk_if_set_event_cb(pfk_pfi_event);
+	pfk_if_iterate(pfk_pfi_attach_one);
+	pfk_net_unlock();
 	err = pfk_hooks_register();
 	if (err)
 		goto err_if;