mirror of
https://github.com/lgcosta/pf-linux-port.git
synced 2026-11-12 22:18:04 -03:00
test(vm): add stress gate 16 and its report runner
Gate 16 asserts invariants under load: states tracked under concurrent connections and drained again, 200 ruleset reloads without failure, 2000 table entries flushed back to zero, 2 MB intact through NAT under a concurrent stream, no kernel faults. Throughput is measured and reported rather than gated, and the traffic phases run at a bounded rate because the PF workqueue backlog exhausts a 1 GB guest above ~1 Gbit/s. The runner writes out/stress/<timestamp>/report.md and captures the guest serial console so a panic stays diagnosable.
This commit is contained in:
lgcosta committed
1 parent
131672d6ce
commit
45ba489672
4 files changed
+506
-1
No files matched your search
@@ -1,7 +1,7 @@
|
||||
# PF-on-Linux top-level make
|
||||
KERNELDIR ?= $(HOME)/lab/linux_kernel/linux-7.3.4
|
||||
|
||||
.PHONY: module tools kernel test clean vm vm-destroy docs demo
|
||||
.PHONY: module tools kernel test stress clean vm vm-destroy docs demo
|
||||
|
||||
module:
|
||||
$(MAKE) -C $(KERNELDIR) M=$(CURDIR)/module modules
|
||||
@@ -21,6 +21,12 @@ vm-destroy:
|
||||
test:
|
||||
./scripts/vm-run-tests.sh
|
||||
|
||||
# Stress: throughput/state/rule/table churn against the lab, with a report in
|
||||
# out/stress/<timestamp>/report.md. Pass options through, e.g.
|
||||
# make stress ARGS="--quick"
|
||||
stress:
|
||||
./scripts/stress/run-stress.sh $(ARGS)
|
||||
|
||||
docs:
|
||||
@for f in docs/en/*.md; do echo "EN $$f"; done
|
||||
@for f in docs/pt-BR/*.md; do echo "PT $$f"; done
|
||||
|
||||
Executable
+118
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env bash
|
||||
# run-stress.sh — drive the PF stress test (tests/vm/16-stress.sh) against the
|
||||
# libvirt lab and turn the metrics into a report.
|
||||
#
|
||||
# ./scripts/stress/run-stress.sh # default profile
|
||||
# ./scripts/stress/run-stress.sh --quick # smaller profile (smoke)
|
||||
# ./scripts/stress/run-stress.sh --seconds 30 --connections 1000
|
||||
#
|
||||
# Output: out/stress/<timestamp>/report.md (+ the raw gate log).
|
||||
# Exit status: 0 when every stress phase passed, 1 otherwise.
|
||||
set -uo pipefail
|
||||
REPO="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
DEB="${DEB:-root@192.168.1.242}"
|
||||
PF0_IP=192.168.122.8
|
||||
TS="$(date +%Y%m%d-%H%M%S)"
|
||||
OUTD="$REPO/out/stress/$TS"
|
||||
mkdir -p "$OUTD"
|
||||
|
||||
SECS=10
|
||||
PAR=4
|
||||
RATE=100M
|
||||
CONNS=300
|
||||
RELOADS=200
|
||||
RULES=200
|
||||
TENTRIES=2000
|
||||
MAXOH=30
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--quick) SECS=5; CONNS=100; RELOADS=50; RULES=50; TENTRIES=500; shift ;;
|
||||
--seconds) SECS=$2; shift 2 ;;
|
||||
--parallel) PAR=$2; shift 2 ;;
|
||||
--rate) RATE=$2; shift 2 ;;
|
||||
--connections) CONNS=$2; shift 2 ;;
|
||||
--reloads) RELOADS=$2; shift 2 ;;
|
||||
--rules) RULES=$2; shift 2 ;;
|
||||
--table-entries) TENTRIES=$2; shift 2 ;;
|
||||
--max-overhead) MAXOH=$2; shift 2 ;;
|
||||
-h|--help) sed -n '2,12p' "$0"; exit 0 ;;
|
||||
*) echo "unknown option: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SSH="ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 -i /root/.ssh/pflab_ed root@$PF0_IP"
|
||||
|
||||
echo "pflab stress: profile seconds=$SECS parallel=$PAR rate/stream=$RATE connections=$CONNS reloads=$RELOADS rules=$RULES table_entries=$TENTRIES max_overhead=$MAXOH%"
|
||||
|
||||
echo "pflab stress: resetting pftest-pf0"
|
||||
ssh "$DEB" "virsh destroy pftest-pf0 >/dev/null 2>&1; virsh start pftest-pf0 >/dev/null" || {
|
||||
echo "pflab stress FAIL: cannot restart pftest-pf0" >&2; exit 1; }
|
||||
up=0
|
||||
for i in $(seq 1 20); do
|
||||
sleep 6
|
||||
if ssh "$DEB" "$SSH true" 2>/dev/null; then up=1; break; fi
|
||||
done
|
||||
[ "$up" = 1 ] || { echo "pflab stress FAIL: pf0 never came back" >&2; exit 1; }
|
||||
|
||||
echo "pflab stress: deploying pf.ko + pfctl to pf0:/opt/pf"
|
||||
for i in 1 2 3; do
|
||||
if scp -q -o ProxyJump=root@192.168.1.242 -o StrictHostKeyChecking=accept-new \
|
||||
-i "$HOME/.ssh/pflab_ed" "$REPO/module/pf.ko" "$REPO/tools/out/pfctl" \
|
||||
"$REPO/tools/out/pdivert" "$REPO/tools/out/pflogd" pf0:/tmp/ 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
ssh "$DEB" "$SSH 'mkdir -p /opt/pf; mv /tmp/pf.ko /tmp/pfctl /tmp/pdivert /tmp/pflogd /opt/pf/ 2>/dev/null; chmod +x /opt/pf/pfctl 2>/dev/null; true'" >/dev/null
|
||||
ssh "$DEB" "$SSH 'test -s /opt/pf/pf.ko && test -x /opt/pf/pfctl'" >/dev/null 2>&1 || {
|
||||
echo "pflab stress FAIL: pf.ko/pfctl missing on pf0" >&2; exit 1; }
|
||||
|
||||
echo "pflab stress: running the stress gate"
|
||||
# The guest's serial console is a pty: a panic or a hang is lost when the VM
|
||||
# reboots. Capture it on the hypervisor (the redirect happens *there*) and
|
||||
# copy it back when the run ends.
|
||||
REMOTE_CONSOLE="/tmp/pf0-console-$TS.log"
|
||||
ssh "$DEB" "PTY=\$(virsh ttyconsole pftest-pf0); setsid nohup timeout 1800 cat \$PTY >$REMOTE_CONSOLE 2>&1 </dev/null &" >/dev/null 2>&1
|
||||
sleep 2
|
||||
export STRESS_SECS="$SECS" STRESS_PARALLEL="$PAR" STRESS_RATE="$RATE" STRESS_CONNS="$CONNS" \
|
||||
STRESS_RELOADS="$RELOADS" STRESS_RULES="$RULES" \
|
||||
STRESS_TABLE_ENTRIES="$TENTRIES" STRESS_MAX_OVERHEAD_PCT="$MAXOH"
|
||||
bash "$REPO/tests/vm/16-stress.sh" >"$OUTD/16-stress.sh.log" 2>&1
|
||||
RC=$?
|
||||
|
||||
# --- report ----------------------------------------------------------------
|
||||
scp -q -o BatchMode=yes -o StrictHostKeyChecking=accept-new "$DEB:$REMOTE_CONSOLE" \
|
||||
"$OUTD/pf0-console.log" 2>/dev/null || true
|
||||
{
|
||||
echo "# PF-on-Linux — stress report"
|
||||
echo
|
||||
echo "- date: $(date -Is)"
|
||||
echo "- profile: seconds=$SECS parallel=$PAR rate/stream=$RATE connections=$CONNS reloads=$RELOADS rules=$RULES table_entries=$TENTRIES max_overhead=${MAXOH}%"
|
||||
echo "- exit status: $RC"
|
||||
echo "- guest serial console: \`pf0-console.log\` (kernel messages during the run)"
|
||||
echo
|
||||
echo "## Phases"
|
||||
echo
|
||||
echo '```'
|
||||
grep '^STRESS ' "$OUTD/16-stress.sh.log" || echo "(no metrics captured)"
|
||||
echo '```'
|
||||
echo
|
||||
echo "## Assertions"
|
||||
echo
|
||||
echo '```'
|
||||
grep -E '^(OK|NOT OK)' "$OUTD/16-stress.sh.log" || echo "(no assertions)"
|
||||
echo '```'
|
||||
} >"$OUTD/report.md"
|
||||
|
||||
echo
|
||||
echo "pflab stress: report -> out/stress/$TS/report.md"
|
||||
grep '^STRESS ' "$OUTD/16-stress.sh.log" || true
|
||||
echo
|
||||
if [ $RC -eq 0 ]; then
|
||||
echo "STRESS RESULT: PASS"
|
||||
else
|
||||
echo "STRESS RESULT: FAIL (rc=$RC)"
|
||||
sed -n '/^NOT OK/p' "$OUTD/16-stress.sh.log" | sed 's/^/ /'
|
||||
fi
|
||||
exit $RC
|
||||
Executable
+280
@@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env bash
|
||||
# M7 gate: stress — the PF engine under sustained load.
|
||||
#
|
||||
# Unlike the functional gates (01-15), this one does not ask "does the packet
|
||||
# arrive?"; it asks "does the engine stay correct, bounded and leak-free while
|
||||
# traffic, state, rules and tables churn?" Phases:
|
||||
#
|
||||
# 0 baseline throughput cli->srv at a bounded rate with the module OUT
|
||||
# of the path
|
||||
# 1 overhead same traffic with PF enabled + stateful rules
|
||||
# 2 state churn 300 concurrent TCP connections; states must appear and
|
||||
# must vanish again (pfctl -F s) -> no state-table leak
|
||||
# 3 rule churn 200 reloads of a 200-rule ruleset, timed on the firewall
|
||||
# 4 table churn 2000 pfr table entries added and flushed
|
||||
# 5 integrity 2 MB through NAT *while* a concurrent stream runs; the
|
||||
# payload must arrive byte-identical (sha256)
|
||||
# 6 health no BUG/WARNING/Oops, counters move, firewall still routes
|
||||
#
|
||||
# Every phase prints `STRESS key=value ...` lines; scripts/stress/run-stress.sh
|
||||
# turns them into a report. Thresholds are overridable from the environment
|
||||
# (STRESS_SECS, STRESS_CONNS, STRESS_RELOADS, STRESS_TABLE_ENTRIES, ...).
|
||||
set -uo pipefail; cd "$(dirname "$0")"; . ./lib.sh
|
||||
|
||||
SECS=${STRESS_SECS:-10}
|
||||
PAR=${STRESS_PARALLEL:-4}
|
||||
RATE=${STRESS_RATE:-100M}
|
||||
CONNS=${STRESS_CONNS:-300}
|
||||
RELOADS=${STRESS_RELOADS:-200}
|
||||
RULES=${STRESS_RULES:-200}
|
||||
TENTRIES=${STRESS_TABLE_ENTRIES:-2000}
|
||||
HOLD=${STRESS_HOLD:-6}
|
||||
MAXOH=${STRESS_MAX_OVERHEAD_PCT:-30}
|
||||
|
||||
# The traffic phases run at a BOUNDED rate ($RATE per stream). This is not
|
||||
# cosmetic: PF's data path steals every skb onto one ordered workqueue
|
||||
# (see docs/*/caveats.md §1), so the backlog — and with it the guest's
|
||||
# memory — grows with the offered rate. Measured on the lab (1 GB guest):
|
||||
# 400 Mbit/s is stable, ~1 Gbit/s OOM-panics the guest. The stress test
|
||||
# therefore works in the regime the port survives and reports the ceiling
|
||||
# separately, instead of killing the firewall under test.
|
||||
|
||||
HELPER=$(pfput ./stress-helper.sh) || { bad "helper upload"; finish; }
|
||||
pf0 "chmod +x $HELPER" >/dev/null
|
||||
|
||||
EGRESS_DEV=$(pf0 "ip route get $SRV_IP | grep -o 'dev [a-z0-9]*' | cut -d' ' -f2")
|
||||
[ -n "$EGRESS_DEV" ] || EGRESS_DEV=eth2
|
||||
EGRESS_IP=$(pf0 "ip -4 addr show $EGRESS_DEV | grep -o 'inet [0-9.]*' | head -1 | cut -d' ' -f2")
|
||||
[ -n "$EGRESS_IP" ] || EGRESS_IP=192.168.122.8
|
||||
|
||||
# --- helpers ---------------------------------------------------------------
|
||||
|
||||
# iperf3 throughput cli->srv, in Mbit/s (one decimal is plenty)
|
||||
iperf_mbit() {
|
||||
local out line val unit
|
||||
out=$(cli "iperf3 -c $SRV_IP -t $1 -P $PAR -b $RATE -i 0 2>/dev/null")
|
||||
line=$(printf '%s\n' "$out" | grep '\[SUM\]' | tail -1)
|
||||
[ -n "$line" ] || { echo 0; return; }
|
||||
val=$(printf '%s\n' "$line" | sed -n 's/.*[^0-9.]\([0-9][0-9.]*\) \([KMG]\?bits\/sec\).*/\1/p')
|
||||
unit=$(printf '%s\n' "$line" | sed -n 's/.* \([KMG]\?bits\/sec\).*/\1/p')
|
||||
[ -n "$val" ] || { echo 0; return; }
|
||||
awk -v v="$val" -v u="$unit" 'BEGIN{
|
||||
if (u ~ /^G/) printf "%.1f\n", v * 1000;
|
||||
else if (u ~ /^M/) printf "%.1f\n", v;
|
||||
else if (u ~ /^K/) printf "%.1f\n", v / 1000;
|
||||
else printf "%.3f\n", v / 1000000; }'
|
||||
}
|
||||
# `pkill -x iperf3` matches the process *name* only. Never use `pkill -f`
|
||||
# with a pattern that also appears in this very command line: it kills the
|
||||
# invoking shell (see the note in 06-rdr.sh).
|
||||
iperf_up() { srv 'pkill -x iperf3 2>/dev/null; iperf3 -s -D >/dev/null 2>&1; sleep 1; pgrep -x iperf3 >/dev/null || echo IPERF_SERVER_DOWN' >/dev/null; }
|
||||
|
||||
# --- phase 0: baseline (module out of the path) ----------------------------
|
||||
pf0 'rmmod pf 2>/dev/null; true' >/dev/null
|
||||
iperf_up
|
||||
BASE=$(iperf_mbit "$SECS")
|
||||
BASE=$(printf '%s' "$BASE" | cut -d. -f1)
|
||||
if [ "${BASE:-0}" -gt 0 ]; then
|
||||
ok "baseline throughput ${BASE} Mbit/s (module unloaded)"
|
||||
else
|
||||
bad "baseline throughput unavailable (iperf3 cli->srv failed)"
|
||||
echo "STRESS phase=baseline baseline_mbit=0 seconds=$SECS parallel=$PAR rate_per_stream=$RATE"
|
||||
finish
|
||||
fi
|
||||
echo "STRESS phase=baseline baseline_mbit=$BASE seconds=$SECS parallel=$PAR rate_per_stream=$RATE"
|
||||
|
||||
# --- phase 1: same traffic with PF enabled ---------------------------------
|
||||
pf0 'insmod /opt/pf/pf.ko 2>/dev/null || insmod /tmp/pf.ko; true' >/dev/null
|
||||
pf0 '/opt/pf/pfctl -e; /opt/pf/pfctl -f /dev/null' >/dev/null
|
||||
|
||||
cat > /tmp/g16.conf <<C
|
||||
pass in quick proto tcp from $CLI_IP to $SRV_IP keep state
|
||||
pass in quick proto icmp from $CLI_IP keep state
|
||||
pass out quick keep state
|
||||
C
|
||||
pfload /tmp/g16.conf || { bad "stress ruleset"; finish; }
|
||||
|
||||
iperf_up
|
||||
PFM=$(iperf_mbit "$SECS")
|
||||
PFM=$(printf '%s' "$PFM" | cut -d. -f1)
|
||||
[ "${PFM:-0}" -gt 0 ] && ok "throughput with PF enabled: ${PFM} Mbit/s" || bad "no throughput with PF enabled"
|
||||
OH=0
|
||||
[ "${BASE:-0}" -gt 0 ] && OH=$(awk -v b="$BASE" -v p="$PFM" 'BEGIN{printf "%d", (b-p)*100/b}')
|
||||
if [ "${OH:-999}" -le "$MAXOH" ]; then
|
||||
ok "PF overhead ${OH}% (<= ${MAXOH}%)"
|
||||
else
|
||||
bad "PF overhead ${OH}% exceeds ${MAXOH}%"
|
||||
fi
|
||||
echo "STRESS phase=overhead pf_mbit=$PFM overhead_pct=$OH limit_pct=$MAXOH rate_per_stream=$RATE"
|
||||
|
||||
# --- phase 2: state churn ---------------------------------------------------
|
||||
# The churn client cannot use sshd: MaxStartups (10:30:100 by default) drops
|
||||
# most concurrent unauthenticated connections. The server therefore gets a
|
||||
# dedicated listener (python's http.server forks per connection and holds it
|
||||
# open until the client closes).
|
||||
#
|
||||
# The setup and cleanup scripts are shipped base64-encoded: a plain `$(...)`
|
||||
# would be expanded by the *intermediate* host (deb), not by the guest, and
|
||||
# would silently do nothing. The listener also outlives the ssh session, so
|
||||
# the setup kills leftovers from an earlier run (an orphan holding the port
|
||||
# would silently break the churn).
|
||||
SETUP=$(cat <<'EOS' | base64 | tr -d '\n'
|
||||
mkdir -p /tmp/stress-web
|
||||
cd /tmp/stress-web || exit 0
|
||||
echo ok > index.html
|
||||
for pid in $(ls /proc 2>/dev/null | grep -E '^[0-9]+$'); do
|
||||
[ "$pid" = "$$" ] && continue
|
||||
cmd=$(tr '\0' ' ' < /proc/$pid/cmdline 2>/dev/null) || continue
|
||||
case "$cmd" in
|
||||
*stress-loop*|*http.server*) kill "$pid" 2>/dev/null ;;
|
||||
esac
|
||||
done
|
||||
if [ -f /tmp/stress-httpd.pid ]; then kill $(cat /tmp/stress-httpd.pid) 2>/dev/null; rm -f /tmp/stress-httpd.pid; fi
|
||||
setsid sh -c 'echo $$ > /tmp/stress-httpd.pid; exec python3 -m http.server 9097 --bind 0.0.0.0' >/dev/null 2>&1 </dev/null &
|
||||
sleep 1
|
||||
EOS
|
||||
)
|
||||
srv "echo $SETUP | base64 -d > /tmp/stress-setup.sh; sh /tmp/stress-setup.sh" >/dev/null
|
||||
if srv 'netstat -lnt 2>/dev/null | grep -q ":9097 "' >/dev/null 2>&1; then
|
||||
ok "churn listener up on srv:9097"
|
||||
else
|
||||
bad "churn listener NOT listening on srv:9097 (stale listener holding the port?)"
|
||||
finish
|
||||
fi
|
||||
RULESET=$(pf0 '/opt/pf/pfctl -s rules | wc -l')
|
||||
PFSTAT=$(pf0 '/opt/pf/pfctl -s info | head -1')
|
||||
pf0 "/opt/pf/pfctl -F s" >/dev/null
|
||||
S0=$(pf0 "$HELPER states")
|
||||
# The connections must stay open while the states are counted: a peer that
|
||||
# closes right away tears the state down before it can be observed.
|
||||
# The iteration list is expanded HERE, on the build host: a `$(seq ...)`
|
||||
# left in the command string would be expanded by the *intermediate* host
|
||||
# (deb), whose newline-separated output breaks the `for` loop one hop later.
|
||||
RANGE=$(seq -s' ' 1 "$CONNS")
|
||||
cli "bash -c 'for i in $RANGE; do (exec 3<>/dev/tcp/$SRV_IP/9097; sleep $HOLD) 2>/dev/null & done; wait'" >/dev/null 2>&1 &
|
||||
CLIPID=$!
|
||||
sleep 4
|
||||
S1=$(pf0 "$HELPER states")
|
||||
EST=$(srv 'netstat -tn 2>/dev/null | grep -c ":9097 "')
|
||||
wait $CLIPID 2>/dev/null
|
||||
if [ "${S1:-0}" -ge $((CONNS / 2)) ]; then
|
||||
ok "state churn: $S1 states tracked for $CONNS connections"
|
||||
else
|
||||
bad "state churn: only $S1 states for $CONNS connections"
|
||||
fi
|
||||
S2=$(pf0 "$HELPER state-reset")
|
||||
sleep 1
|
||||
S3=$(pf0 "$HELPER states")
|
||||
if [ "${S3:-0}" -le "${S0:-0}" ]; then
|
||||
ok "state table drained after reset ($S0 -> $S3)"
|
||||
else
|
||||
bad "state table did not drain ($S0 -> $S3)"
|
||||
fi
|
||||
echo "STRESS phase=state_churn connections=$CONNS states_before=$S0 states_created=$S1 states_after_reset=$S3 established_on_srv=${EST:-0} rules_loaded=$RULESET pf_status=$(printf '%s' "$PFSTAT" | tr -s ' ' | cut -d' ' -f2-)"
|
||||
|
||||
# --- phase 3: rule churn ---------------------------------------------------
|
||||
{
|
||||
echo "table <stress> { }"
|
||||
echo "pass in quick proto tcp from $CLI_IP to $SRV_IP keep state"
|
||||
echo "pass out quick keep state"
|
||||
i=1
|
||||
while [ "$i" -le "$RULES" ]; do
|
||||
# a distinct destination port per rule: rules with the same shape
|
||||
# are folded by pfctl's optimizer into one automatic table, which
|
||||
# would make the rule count meaningless
|
||||
echo "block drop in quick proto tcp from 10.100.$((i % 250)).$((i % 250 + 1)) to any port $((1000 + i))"
|
||||
i=$((i + 1))
|
||||
done
|
||||
} > /tmp/g16-churn.conf
|
||||
CH=$(pfput /tmp/g16-churn.conf) || { bad "churn ruleset upload"; finish; }
|
||||
pf0 "/opt/pf/pfctl -f $CH" >/dev/null || { bad "churn ruleset load"; finish; }
|
||||
R0=$(pf0 "$HELPER rules")
|
||||
RL=$(pf0 "$HELPER reload-loop $RELOADS $CH")
|
||||
eval "$RL"
|
||||
if [ "${reload_fails:-1}" -eq 0 ]; then
|
||||
ok "rule churn: $RELOADS reloads, no failure (avg ${reload_ms_avg:-?} ms, max ${reload_ms_max:-?} ms)"
|
||||
else
|
||||
bad "rule churn: ${reload_fails:-?} of $RELOADS reloads failed"
|
||||
fi
|
||||
R1=$(pf0 "$HELPER rules")
|
||||
if [ "${R1:-0}" -eq "${R0:-0}" ] && [ "${R1:-0}" -eq "$((RULES + 2))" ]; then
|
||||
ok "rule count stable across churn ($R1 rules)"
|
||||
else
|
||||
bad "rule count drifted: $R0 -> $R1 (expected $((RULES + 2)))"
|
||||
fi
|
||||
echo "STRESS phase=rule_churn reloads=$RELOADS rules_per_load=$RULES reload_fails=${reload_fails:-0} reload_ms_avg=${reload_ms_avg:-0} reload_ms_max=${reload_ms_max:-0} rules_final=$R1"
|
||||
|
||||
# --- phase 4: table churn --------------------------------------------------
|
||||
TC=$(pf0 "$HELPER table-churn $TENTRIES stress")
|
||||
eval "$TC"
|
||||
if [ "${table_add_errors:-1}" -eq 0 ] && [ "${table_entries_after_add:-0}" -ge $((TENTRIES / 2)) ]; then
|
||||
ok "table churn: ${table_entries_after_add} entries added"
|
||||
else
|
||||
bad "table churn: ${table_entries_after_add:-0} entries, ${table_add_errors:-?} errors"
|
||||
fi
|
||||
if [ "${table_entries_after_flush:-1}" -eq 0 ]; then
|
||||
ok "table flushed back to 0 entries"
|
||||
else
|
||||
bad "table not empty after flush (${table_entries_after_flush})"
|
||||
fi
|
||||
echo "STRESS phase=table_churn entries_requested=$TENTRIES entries_added=${table_entries_after_add:-0} add_errors=${table_add_errors:-0} entries_after_flush=${table_entries_after_flush:-0}"
|
||||
|
||||
# --- phase 5: integrity under load -----------------------------------------
|
||||
cat > /tmp/g16-nat.conf <<C
|
||||
pass out quick on $EGRESS_DEV from $CLI_IP to any nat-to ($EGRESS_DEV)
|
||||
pass in quick proto tcp from $CLI_IP to $SRV_IP port 9099 keep state
|
||||
pass out quick keep state
|
||||
C
|
||||
pfload /tmp/g16-nat.conf || { bad "nat ruleset"; finish; }
|
||||
|
||||
SZ=2000000
|
||||
srv 'pkill nc 2>/dev/null; head -c 2000000 /dev/zero | tr "\0" A > /tmp/stress.bin; sleep 1; nohup setsid nc -l -p 9099 < /tmp/stress.bin >/dev/null 2>&1 & sleep 1' >/dev/null
|
||||
SRC_SUM=$(srv 'sha256sum /tmp/stress.bin | cut -c1-64' | tr -d '[:space:]')
|
||||
# saturating background stream while the payload crosses PF
|
||||
cli "iperf3 -c $SRV_IP -t 15 -P $PAR -b $RATE >/dev/null 2>&1 &" >/dev/null 2>&1
|
||||
sleep 1
|
||||
GOT=$(cli 'nc -w20 10.0.2.20 9099 | sha256sum | cut -c1-64' | tr -d '[:space:]')
|
||||
if [ -n "$SRC_SUM" ] && [ "$GOT" = "$SRC_SUM" ]; then
|
||||
ok "2 MB through NAT intact under load (sha256 match)"
|
||||
INTEG=yes
|
||||
else
|
||||
bad "payload corrupted under load (src=$SRC_SUM got=$GOT)"
|
||||
INTEG=no
|
||||
fi
|
||||
echo "STRESS phase=integrity bytes=$SZ sha256_match=$INTEG"
|
||||
|
||||
# --- phase 6: health -------------------------------------------------------
|
||||
ANOM=$(pf0 "$HELPER dmesg")
|
||||
if [ -z "$ANOM" ]; then
|
||||
ok "no kernel BUG/WARN/Oops after the stress run"
|
||||
else
|
||||
bad "dmesg anomalies:"
|
||||
printf '%s\n' "$ANOM" | sed 's/^/ /'
|
||||
fi
|
||||
MATCHED=$(pf0 "$HELPER counter match")
|
||||
ENTRIES=$(pf0 "$HELPER state-entries")
|
||||
[ "${MATCHED:-0}" -gt 0 ] && ok "PF counters moved (match=${MATCHED:-0} state_entries=${ENTRIES:-0})" \
|
||||
|| bad "PF counters did not move (match=${MATCHED:-0})"
|
||||
cli "ping -c 3 -W 2 $SRV_IP >/dev/null 2>&1" && ok "firewall still routes traffic after stress" \
|
||||
|| bad "firewall stopped routing traffic after stress"
|
||||
echo "STRESS phase=health dmesg_anomalies=$(printf '%s' "$ANOM" | grep -c . ) match=${MATCHED:-0} state_entries=${ENTRIES:-0}"
|
||||
|
||||
# --- cleanup ---------------------------------------------------------------
|
||||
CLEAN=$(cat <<'EOS' | base64 | tr -d '\n'
|
||||
if [ -f /tmp/stress-httpd.pid ]; then kill $(cat /tmp/stress-httpd.pid) 2>/dev/null; rm -f /tmp/stress-httpd.pid; fi
|
||||
for pid in $(ls /proc 2>/dev/null | grep -E '^[0-9]+$'); do
|
||||
[ "$pid" = "$$" ] && continue
|
||||
cmd=$(tr '\0' ' ' < /proc/$pid/cmdline 2>/dev/null) || continue
|
||||
case "$cmd" in
|
||||
*http.server*) kill "$pid" 2>/dev/null ;;
|
||||
esac
|
||||
done
|
||||
pkill -x nc 2>/dev/null
|
||||
pkill -x iperf3 2>/dev/null
|
||||
true
|
||||
EOS
|
||||
)
|
||||
srv "echo $CLEAN | base64 -d > /tmp/stress-clean.sh; sh /tmp/stress-clean.sh" >/dev/null
|
||||
cli 'pkill -x iperf3 2>/dev/null; true' >/dev/null
|
||||
pf0 '/opt/pf/pfctl -d; /opt/pf/pfctl -f /dev/null' >/dev/null
|
||||
finish
|
||||
Executable
+101
@@ -0,0 +1,101 @@
|
||||
#!/bin/sh
|
||||
# Helper executed ON pf0 by 16-stress.sh (uploaded with pfput to avoid quoting
|
||||
# hell through two ssh hops). POSIX sh: the guest is Alpine (busybox ash).
|
||||
#
|
||||
# $1 = "slab" | "dmesg" | "states" | "state-reset" | "rules" | "info"
|
||||
# | "reload-loop <n> <ruleset>" | "table-churn <n> <table>"
|
||||
#
|
||||
# Every subcommand prints machine-readable KEY=VALUE pairs on stdout so the
|
||||
# gate can parse them without another ssh round trip.
|
||||
PFCTL=/opt/pf/pfctl
|
||||
|
||||
now_ms() { awk '{printf "%d", $1 * 1000}' /proc/uptime; }
|
||||
|
||||
case "${1:-}" in
|
||||
slab)
|
||||
grep -E '^(pf|pfk)' /proc/slabinfo | tr -s ' ' | cut -d' ' -f1,3 | sort
|
||||
;;
|
||||
dmesg)
|
||||
# the CPU's Speculative Return Stack Overflow advisory is a kernel
|
||||
# build/CPU feature notice, not a fault of this module
|
||||
dmesg | grep -E 'BUG:|WARNING:|Oops|general protection|kernel BUG' \
|
||||
| grep -v 'Speculative Return Stack Overflow' || true
|
||||
;;
|
||||
states)
|
||||
# `pfctl -s states` prints one line per direction, so count the
|
||||
# initiator lines ("->") to get the number of states
|
||||
$PFCTL -s states 2>/dev/null | grep -c ' -> ' || true
|
||||
;;
|
||||
state-reset)
|
||||
# -F s clears the state table (-s s only *shows* it)
|
||||
$PFCTL -F s >/dev/null 2>&1 || true
|
||||
$PFCTL -s states 2>/dev/null | grep -c ' -> ' || true
|
||||
;;
|
||||
rules)
|
||||
$PFCTL -s rules 2>/dev/null | grep -c . || true
|
||||
;;
|
||||
info)
|
||||
$PFCTL -s info 2>/dev/null
|
||||
;;
|
||||
counter)
|
||||
# $2 = counter name as printed by `pfctl -s info` (match, short,
|
||||
# state-mismatch, ...). Prints the value, or 0 when absent.
|
||||
$PFCTL -s info 2>/dev/null | awk -v k="${2:-match}" '$1 == k { print $2; exit }'
|
||||
;;
|
||||
state-entries)
|
||||
# the number of entries the state table currently holds
|
||||
$PFCTL -s info 2>/dev/null | awk '/current entries/ { print $NF; exit }'
|
||||
;;
|
||||
reload-loop)
|
||||
# $2 = iterations, $3 = ruleset file. Times each `pfctl -f` on the
|
||||
# firewall itself (no ssh overhead in the measurement).
|
||||
n="${2:-0}"; f="${3:-/tmp/g16-churn.conf}"
|
||||
i=0; fails=0; min=999999999; max=0; sum=0
|
||||
while [ "$i" -lt "$n" ]; do
|
||||
s=$(now_ms)
|
||||
if ! $PFCTL -f "$f" >/dev/null 2>&1; then
|
||||
fails=$((fails + 1))
|
||||
fi
|
||||
e=$(now_ms)
|
||||
d=$((e - s)); [ "$d" -lt 0 ] && d=0
|
||||
sum=$((sum + d))
|
||||
[ "$d" -lt "$min" ] && min=$d
|
||||
[ "$d" -gt "$max" ] && max=$d
|
||||
i=$((i + 1))
|
||||
done
|
||||
[ "$min" = 999999999 ] && min=0
|
||||
avg=0; [ "$n" -gt 0 ] && avg=$((sum / n))
|
||||
echo "reload_iterations=$n reload_fails=$fails reload_ms_min=$min reload_ms_avg=$avg reload_ms_max=$max"
|
||||
;;
|
||||
table-churn)
|
||||
# $2 = number of addresses, $3 = table name. Adds N distinct addresses,
|
||||
# counts the table, flushes it and counts again.
|
||||
n="${2:-0}"; t="${3:-stress}"
|
||||
$PFCTL -t "$t" -T flush >/dev/null 2>&1 || true
|
||||
i=1; err=0
|
||||
while [ "$i" -le "$n" ]; do
|
||||
# 10.0.<a>.<b> keeps every entry distinct without touching the
|
||||
# lab's real segments (10.0.1.0/24 lan, 10.0.2.0/24 wan)
|
||||
a=$(( (i - 1) / 250 + 100 ))
|
||||
b=$(( (i - 1) % 250 + 1 ))
|
||||
if ! $PFCTL -t "$t" -T add "10.$a.$b" >/dev/null 2>&1; then
|
||||
err=$((err + 1))
|
||||
fi
|
||||
i=$((i + 1))
|
||||
done
|
||||
added=$($PFCTL -t "$t" -T show 2>/dev/null | grep -c . || true)
|
||||
$PFCTL -t "$t" -T flush >/dev/null 2>&1 || true
|
||||
left=$($PFCTL -t "$t" -T show 2>/dev/null | grep -c . || true)
|
||||
if [ "${left:-0}" -ne 0 ]; then
|
||||
# flush did not empty the table: fall back to deleting every entry
|
||||
$PFCTL -t "$t" -T show 2>/dev/null | awk '{print $1}' | \
|
||||
while read -r a; do $PFCTL -t "$t" -T delete "$a" >/dev/null 2>&1 || true; done
|
||||
left=$($PFCTL -t "$t" -T show 2>/dev/null | grep -c . || true)
|
||||
fi
|
||||
echo "table_entries_requested=$n table_add_errors=$err table_entries_after_add=$added table_entries_after_flush=$left"
|
||||
;;
|
||||
*)
|
||||
echo "unknown helper command: ${1:-}" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
Reference in new issue
Block a user