nat: re-route inbound packets whose destination pf rewrote (reverse NAT) instead of delivering them locally

module/pfsrc/pfk_hooks.c

@@ -71,14 +71,50 @@ pfk_pkt_reinject(struct pfk_pkt *pk)
 	kfree(pk);
 }
 
+
+/*
+ * Reverse NAT: pf rewrote the destination of an inbound packet (the reply
+ * of a translated flow).  OpenBSD re-routes such a packet in ip_input()
+ * (PF_TAG_REROUTE); Linux has no such layer, so re-route it here and push
+ * it out through the new egress device.
+ */
+static void
+pfk_reroute(struct sk_buff *skb)
+{
+	struct flowi4	 fl4 = { };
+	struct rtable	*rt;
+	struct iphdr	*iph = ip_hdr(skb);
+
+	fl4.daddr = iph->daddr;
+	fl4.saddr = iph->saddr;
+	fl4.flowi4_oif = 0;
+	fl4.flowi4_mark = 0;
+	fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
+
+	rt = ip_route_output_key(&init_net, &fl4);
+	if (IS_ERR(rt) || rt == NULL) {
+		kfree_skb(skb);
+		return;
+	}
+	skb_dst_set(skb, &rt->dst);
+	skb->dev = rt->dst.dev;
+	skb_reset_network_header(skb);
+	rt->dst.output(&init_net, skb->sk, skb);
+}
+
 static void
 pfk_pkt_work(struct work_struct *w)
 {
 	struct pfk_pkt *pk = container_of(w, struct pfk_pkt, work);
 	struct mbuf *m, *m0;
 	struct sk_buff *orig;
+	__be32 orig_dst = 0;
 	int action;
 
+	if (pk->af == AF_INET && pk->dir == PF_IN &&
+	    pskb_may_pull(pk->skb, sizeof(struct iphdr)))
+		orig_dst = ip_hdr(pk->skb)->daddr;
+
 	m = pfk_m_wrap_skb(pk->skb, pk->af);
 	if (!m) {
 		pfk_pkt_reinject(pk);	/* like "pf not consulted" */
@@ -98,6 +134,16 @@ pfk_pkt_work(struct work_struct *w)
 		/* same buffer, possibly rewritten in place: our wrapper
 		 * held one extra skb reference — release it */
 		m_freem(m);
+		if (action == PF_PASS && orig_dst != 0 &&
+		    pk->af == AF_INET && pk->dir == PF_IN &&
+		    pskb_may_pull(pk->skb, sizeof(struct iphdr)) &&
+		    ip_hdr(pk->skb)->daddr != orig_dst) {
+			/* pf translated the destination (reverse NAT):
+			 * re-route instead of delivering locally */
+			pfk_reroute(pk->skb);
+			kfree(pk);
+			return;
+		}
 		if (action == PF_PASS) {
 			pfk_pkt_reinject(pk);
 			return;

tests/vm/05-nat.sh

@@ -17,8 +17,11 @@ EGRESS_IP=$(pf0 "ip -4 addr show $EGRESS_DEV | grep -o 'inet [0-9.]*' | head -1
 pf0 'insmod /opt/pf/pf.ko 2>/dev/null || insmod /tmp/pf.ko; true'
 pf0 '/opt/pf/pfctl -e; /opt/pf/pfctl -f /dev/null' >/dev/null
 
+# OpenBSD expands "(egress)" (an interface *group*) into the addresses of all
+# its members - on a multi-NIC host that is ambiguous, so the gate uses the
+# concrete transit interface (the OpenBSD-recommended form for NAT pools).
 cat > /tmp/g05.conf <<C
-pass out quick on egress from 10.0.1.0/24 to any nat-to (egress)
+pass out quick on egress from 10.0.1.0/24 to any nat-to ($EGRESS_DEV)
 pass in quick on ingress proto icmp keep state
 pass out quick on egress proto icmp keep state
 C