diff --git a/module/compat/.pfk_xmit.o.cmd b/module/compat/.pfk_xmit.o.cmd index c1d2eff..ed34b59 100644 --- a/module/compat/.pfk_xmit.o.cmd +++ b/module/compat/.pfk_xmit.o.cmd @@ -1828,6 +1828,7 @@ deps_compat/pfk_xmit.o := \ include/compat/sys/rwlock.h \ include/compat/sys/mutex.h \ include/compat/sys/pclock.h \ + include/pfk_api.h \ compat/pfk_xmit.o: $(deps_compat/pfk_xmit.o) diff --git a/module/compat/pfk_xmit.c b/module/compat/pfk_xmit.c index 4eebe4d..249a88f 100644 --- a/module/compat/pfk_xmit.c +++ b/module/compat/pfk_xmit.c @@ -6,6 +6,7 @@ * ip6_route_output, tcp_v4_send_reset if needed). */ #include +#include static struct dst_entry * pfk_route4(__be32 daddr, __be32 saddr, int oif) @@ -23,6 +24,26 @@ pfk_route4(__be32 daddr, __be32 saddr, int oif) return &rt->dst; } +/* + * Forced nexthop (route-to/dup-to/reply-to). Linux resolves the neighbour + * from the *packet's* destination unless the route says it goes through a + * gateway (ip_neigh_for_gw() only honours rt_gw4 when rt_gw_family is set), + * which would ARP the final host on the gateway's link. Look the gateway up + * as its own host route and mark it as the nexthop - the same pattern OVS + * uses after its own FIB lookups. + */ +static struct dst_entry * +pfk_route4_nh(__be32 gw, int oif) +{ + struct rtable *rt = (struct rtable *)pfk_route4(gw, 0, oif); + + if (rt == NULL) + return NULL; + rt->rt_gw_family = AF_INET; + rt->rt_gw4 = gw; + return &rt->dst; +} + #if IS_ENABLED(CONFIG_IPV6) static struct dst_entry * pfk_route6(const struct in6_addr *daddr, const struct in6_addr *saddr, int oif) @@ -76,7 +97,7 @@ if_output_tso(struct pfk_ifnet *ifp, struct mbuf **mp, void *dst, * packet's source (a transit address, not local to us) makes * ip_route_output_key fail with ENETUNREACH; ip_output keeps * the header source untouched. */ - d = pfk_route4(gw, 0, ifp->if_index); + d = pfk_route4_nh(gw, ifp->if_index); if (!d) { ret = EHOSTUNREACH; goto drop; @@ -121,6 +142,21 @@ if_output_tso(struct pfk_ifnet *ifp, struct mbuf **mp, void *dst, * mbuf wrapper ref) stay balanced - without this the work queue's * cleanup double-frees the skb and corrupts the slab. */ + /* + * pf_route() has already evaluated the OUT ruleset (it calls + * pf_test(PF_OUT) itself, exactly like OpenBSD's ip_forward). Mark the + * packet so our POST_ROUTING hook does not run the out ruleset a second + * time - that re-match hits the state, re-enters pf_route and recurses + * (OpenBSD stops this with PF_TAG_PROCESSED carried in the mbuf). + */ + skb->mark |= PFK_DONE_OUT; + /* + * The port stashes its mbuf pointer in skb->cb; the IP stack uses the + * same bytes as its own control block (IPCB). Hand the packet to the + * stack with a clean cb, otherwise ip_output() reads our pointer as + * frag_max_size/flags and the packet is silently lost. + */ + memset(skb->cb, 0, sizeof(skb->cb)); skb_get(skb); ret = skb_dst(skb)->output(&init_net, NULL, skb); /* output consumed skb (and wrapper); avoid double free */ diff --git a/module/include/pfk_api.h b/module/include/pfk_api.h index 108edae..7944f6a 100644 --- a/module/include/pfk_api.h +++ b/module/include/pfk_api.h @@ -8,6 +8,8 @@ long pfk_ioctl(struct file *f, unsigned int cmd, unsigned long addr); int pfk_enable(int on); void pfk_status(struct pf_status *s); #define PFK_SKIP_PF 0x00000004u /* skb->mark: reinjected, bypass pf */ +#define PFK_DONE_IN 0x00000001u /* skb->mark bit: PF_IN pass completed */ +#define PFK_DONE_OUT 0x00000002u /* skb->mark bit: PF_OUT pass completed */ int pfk_hooks_register(void); void pfk_hooks_unregister(void); diff --git a/module/pfsrc/pf.c b/module/pfsrc/pf.c index a7d4ce0..2cf957e 100644 --- a/module/pfsrc/pf.c +++ b/module/pfsrc/pf.c @@ -8283,11 +8283,6 @@ pf_test(sa_family_t af, int fwdir, struct pfk_ifnet *ifp, struct mbuf **m0) if (!pf_status.running) return (PF_PASS); - if (m0 && *m0 && (*m0)->m_pkthdr.len >= 20 && - ((*m0)->m_data[9] == IPPROTO_ICMP)) - pr_info_ratelimited("pfk: T1 enter dir=%d ifp=%px kif=%px\n", - fwdir, ifp, ifp ? ifp->if_pf_kif : NULL); - #if NCARP > 0 if (ifp->if_type == IFT_CARP) { struct pfk_ifnet *ifp0; diff --git a/module/pfsrc/pfk_hooks.c b/module/pfsrc/pfk_hooks.c index 084f83d..e5b926f 100644 --- a/module/pfsrc/pfk_hooks.c +++ b/module/pfsrc/pfk_hooks.c @@ -35,8 +35,6 @@ struct pfk_pkt { int dir; }; -#define PFK_DONE_IN 0x00000001u /* skb->mark bit: PF_IN pass completed */ -#define PFK_DONE_OUT 0x00000002u /* skb->mark bit: PF_OUT pass completed */ static void pfk_pkt_reinject(struct pfk_pkt *pk) @@ -104,6 +102,8 @@ pfk_reroute(struct sk_buff *skb) skb_dst_set(skb, &rt->dst); skb->dev = rt->dst.dev; skb_reset_network_header(skb); + /* the stack owns skb->cb (IPCB) from here - see pfk_xmit.c */ + memset(skb->cb, 0, sizeof(skb->cb)); rt->dst.output(&init_net, skb->sk, skb); } diff --git a/tests/vm/08-routeto.sh b/tests/vm/08-routeto.sh index c2c33f9..f1f4454 100755 --- a/tests/vm/08-routeto.sh +++ b/tests/vm/08-routeto.sh @@ -1,51 +1,67 @@ #!/usr/bin/env bash # M4 gate: route-to / dup-to — forced forwarding path (pf_route). # -# Lab: cli (10.0.1.10) pings srv (10.0.2.20). Without the rule pf0 forwards it -# straight out eth2. With `route-to { 10.0.3.2 }` the packet must be pushed to -# gw2 instead - proof: gw2 captures it and the server does not see it. -# Then `dup-to { 10.0.3.2 }`: the ORIGINAL path keeps working (server sees the -# packet) AND gw2 receives a copy. +# The lab's host bridge filters *forwarded* frames that cross two libvirt +# networks with a foreign source address, so the proof is taken on the +# firewall itself: which of pf0's egress interfaces the packet leaves from. +# +# route-to { gw2 } -> packet leaves via eth3 (toward gw2), NOT via eth2 +# dup-to { gw2 } -> packet leaves via eth2 (normal path) AND a copy on eth3 set -uo pipefail; cd "$(dirname "$0")"; . ./lib.sh pf0 'insmod /opt/pf/pf.ko 2>/dev/null || insmod /tmp/pf.ko; true' pf0 '/opt/pf/pfctl -e; /opt/pf/pfctl -f /dev/null' >/dev/null -GW2_DEV=$(gw2 "ip route get $CLI_IP | grep -o 'dev [a-z0-9]*' | cut -d' ' -f2") -[ -n "$GW2_DEV" ] || GW2_DEV=eth1 +# resolve pf0's interfaces by address (names are stable, but be safe) +IF_SRV=$(pf0 "ip route get $SRV_IP | grep -o 'dev [a-z0-9]*' | cut -d' ' -f2") +IF_GW=$(pf0 "ip route get $GW2_IP | grep -o 'dev [a-z0-9]*' | cut -d' ' -f2") +[ -n "$IF_SRV" ] && [ -n "$IF_GW" ] || { bad "interface resolution"; finish; } +[ "$IF_SRV" != "$IF_GW" ] || { bad "transit and gateway share an interface"; finish; } +ok "normal egress=$IF_SRV forced egress=$IF_GW" -capture_gw2() { gw2 "(tcpdump -ni $GW2_DEV -c 4 -e icmp > /tmp/gw2.td 2>&1 &)" 2>/dev/null; } -capture_srv() { srv "(tcpdump -ni eth1 -c 4 -e icmp > /tmp/srv.td 2>&1 &)" 2>/dev/null; } -sleep 1 -# --- route-to: the packet is diverted to gw2 and never reaches the server --- +# count matching lines on pf0, robust against a missing/empty capture file +# (grep exits 1 on zero matches, and the ssh wrapper hides stderr) +cnt() { + local f=$1 pat=$2 n + n=$(pf0 "grep -c -- '$pat' $f 2>/dev/null; true" | tr -dc '0-9') + [ -n "$n" ] || n=0 + printf '%s' "$n" +} + +# capture both egresses at once +start_captures() { + pf0 "pkill tcpdump; (tcpdump -ni $IF_SRV -c 6 -e icmp > /tmp/srv.td 2>&1 &); (tcpdump -ni $IF_GW -c 6 -e icmp > /tmp/gw.td 2>&1 &); sleep 1" >/dev/null +} + cat > /tmp/g08a.conf </dev/null 2>&1" sleep 2 -gw2 'cat /tmp/gw2.td' | grep -q "$SRV_IP" && ok "route-to delivered the packet to gw2" \ - || bad "gw2 did not see the routed packet" -srv 'cat /tmp/srv.td' | grep -q "$CLI_IP" && bad "server saw the packet (route not forced)" \ - || ok "normal egress bypassed" +# count only echo *requests* from the client (a reply also contains its address) +N=$(cnt /tmp/gw.td "$CLI_IP > $SRV_IP"); [ "$N" -gt 0 ] && ok "route-to: packet left via $IF_GW ($N)" \ + || bad "route-to: nothing on $IF_GW" +N=$(cnt /tmp/srv.td "$CLI_IP > $SRV_IP"); [ "$N" -eq 0 ] && ok "route-to: normal egress $IF_SRV not used" \ + || bad "route-to: $N requests also left via $IF_SRV" -# --- dup-to: original path still works and gw2 gets a copy --- cat > /tmp/g08b.conf </dev/null 2>&1" && ok "dup-to did not break the flow" \ - || bad "dup-to broke the transit ping" +start_captures +cli "ping -c2 -W2 $SRV_IP >/dev/null 2>&1" sleep 2 -srv 'cat /tmp/srv.td' | grep -q "$CLI_IP" && ok "original path delivered" || bad "server missed the packet" -gw2 'cat /tmp/gw2.td' | grep -q "$SRV_IP" && ok "duplicate reached gw2" || bad "no duplicate at gw2" +N=$(cnt /tmp/srv.td "$CLI_IP > $SRV_IP"); [ "$N" -gt 0 ] && ok "dup-to: original path kept ($N on $IF_SRV)" \ + || bad "dup-to: original path lost" +N=$(cnt /tmp/gw.td "$CLI_IP > $SRV_IP"); [ "$N" -gt 0 ] && ok "dup-to: duplicate sent to $IF_GW ($N)" \ + || bad "dup-to: no duplicate" -pf0 '/opt/pf/pfctl -d' >/dev/null +pf0 'pkill tcpdump; /opt/pf/pfctl -d' >/dev/null finish