route-to: forced nexthop via rt_gw4/rt_gw_family (Linux resolves neighbours from the gateway only when the route says so); cleanup of debug instrumentation

module/compat/.pfk_xmit.o.cmd

@@ -1828,6 +1828,7 @@ deps_compat/pfk_xmit.o := \
   include/compat/sys/rwlock.h \
   include/compat/sys/mutex.h \
   include/compat/sys/pclock.h \
+  include/pfk_api.h \
 
 compat/pfk_xmit.o: $(deps_compat/pfk_xmit.o)
 

module/compat/pfk_xmit.c

@@ -6,6 +6,7 @@
  * ip6_route_output, tcp_v4_send_reset if needed).
  */
 #include <pfk.h>
+#include <pfk_api.h>
 
 static struct dst_entry *
 pfk_route4(__be32 daddr, __be32 saddr, int oif)
@@ -23,6 +24,26 @@ pfk_route4(__be32 daddr, __be32 saddr, int oif)
 	return &rt->dst;
 }
 
+/*
+ * Forced nexthop (route-to/dup-to/reply-to).  Linux resolves the neighbour
+ * from the *packet's* destination unless the route says it goes through a
+ * gateway (ip_neigh_for_gw() only honours rt_gw4 when rt_gw_family is set),
+ * which would ARP the final host on the gateway's link.  Look the gateway up
+ * as its own host route and mark it as the nexthop - the same pattern OVS
+ * uses after its own FIB lookups.
+ */
+static struct dst_entry *
+pfk_route4_nh(__be32 gw, int oif)
+{
+	struct rtable *rt = (struct rtable *)pfk_route4(gw, 0, oif);
+
+	if (rt == NULL)
+		return NULL;
+	rt->rt_gw_family = AF_INET;
+	rt->rt_gw4 = gw;
+	return &rt->dst;
+}
+
 #if IS_ENABLED(CONFIG_IPV6)
 static struct dst_entry *
 pfk_route6(const struct in6_addr *daddr, const struct in6_addr *saddr, int oif)
@@ -76,7 +97,7 @@ if_output_tso(struct pfk_ifnet *ifp, struct mbuf **mp, void *dst,
 		 * packet's source (a transit address, not local to us) makes
 		 * ip_route_output_key fail with ENETUNREACH; ip_output keeps
 		 * the header source untouched. */
-		d = pfk_route4(gw, 0, ifp->if_index);
+		d = pfk_route4_nh(gw, ifp->if_index);
 		if (!d) {
 			ret = EHOSTUNREACH;
 			goto drop;
@@ -121,6 +142,21 @@ if_output_tso(struct pfk_ifnet *ifp, struct mbuf **mp, void *dst,
 	 * mbuf wrapper ref) stay balanced - without this the work queue's
 	 * cleanup double-frees the skb and corrupts the slab.
 	 */
+	/*
+	 * pf_route() has already evaluated the OUT ruleset (it calls
+	 * pf_test(PF_OUT) itself, exactly like OpenBSD's ip_forward).  Mark the
+	 * packet so our POST_ROUTING hook does not run the out ruleset a second
+	 * time - that re-match hits the state, re-enters pf_route and recurses
+	 * (OpenBSD stops this with PF_TAG_PROCESSED carried in the mbuf).
+	 */
+	skb->mark |= PFK_DONE_OUT;
+	/*
+	 * The port stashes its mbuf pointer in skb->cb; the IP stack uses the
+	 * same bytes as its own control block (IPCB).  Hand the packet to the
+	 * stack with a clean cb, otherwise ip_output() reads our pointer as
+	 * frag_max_size/flags and the packet is silently lost.
+	 */
+	memset(skb->cb, 0, sizeof(skb->cb));
 	skb_get(skb);
 	ret = skb_dst(skb)->output(&init_net, NULL, skb);
 	/* output consumed skb (and wrapper); avoid double free */

module/include/pfk_api.h

@@ -8,6 +8,8 @@ long pfk_ioctl(struct file *f, unsigned int cmd, unsigned long addr);
 int  pfk_enable(int on);
 void pfk_status(struct pf_status *s);
 #define PFK_SKIP_PF	0x00000004u	/* skb->mark: reinjected, bypass pf */
+#define PFK_DONE_IN	0x00000001u	/* skb->mark bit: PF_IN pass completed */
+#define PFK_DONE_OUT	0x00000002u	/* skb->mark bit: PF_OUT pass completed */
 
 int  pfk_hooks_register(void);
 void pfk_hooks_unregister(void);

module/pfsrc/pf.c

@@ -8283,11 +8283,6 @@ pf_test(sa_family_t af, int fwdir, struct pfk_ifnet *ifp, struct mbuf **m0)
 
 	if (!pf_status.running)
 		return (PF_PASS);
-	if (m0 && *m0 && (*m0)->m_pkthdr.len >= 20 &&
-	    ((*m0)->m_data[9] == IPPROTO_ICMP))
-		pr_info_ratelimited("pfk: T1 enter dir=%d ifp=%px kif=%px\n",
-		    fwdir, ifp, ifp ? ifp->if_pf_kif : NULL);
-
 #if NCARP > 0
 	if (ifp->if_type == IFT_CARP) {
 		struct pfk_ifnet *ifp0;

module/pfsrc/pfk_hooks.c

@@ -35,8 +35,6 @@ struct pfk_pkt {
 	int			dir;
 };
 
-#define PFK_DONE_IN	0x00000001u	/* skb->mark bit: PF_IN pass completed */
-#define PFK_DONE_OUT	0x00000002u	/* skb->mark bit: PF_OUT pass completed */
 
 static void
 pfk_pkt_reinject(struct pfk_pkt *pk)
@@ -104,6 +102,8 @@ pfk_reroute(struct sk_buff *skb)
 	skb_dst_set(skb, &rt->dst);
 	skb->dev = rt->dst.dev;
 	skb_reset_network_header(skb);
+	/* the stack owns skb->cb (IPCB) from here - see pfk_xmit.c */
+	memset(skb->cb, 0, sizeof(skb->cb));
 	rt->dst.output(&init_net, skb->sk, skb);
 }