[PATCH] netfilter: TCPMSS: fix dropped packets when MSS option is unaligned

From: Kacper Kokot

Date: Mon May 25 2026 - 16:12:05 EST


Padding TCP options with NOPs is optional, so it is legal to send an
MSS option that is not aligned to a word boundary and therefore not
aligned for checksum calculation. The current TCPMSS target is not
robust to this: when the MSS option is unaligned it produces an
invalid checksum, and the packet is dropped.

When the changed word is not aligned, the modified bytes straddle two
checksum words, and using the standard incremental update helper
(which assumes alignment) produces an invalid checksum:

| w1 | w2 |
OLD | a b | c d |
NEW | a b' | c' d |

Since b' and c' sit across w1 and w2, we could compute the incremental
checksum in two operations by recalculating w1 and then w2:

C' = C - w1 + w1' - w2 + w2'

But working it out:

C' = C - w1 - w2 + w1' + w2'
= C - (a * 2^8 + b) - (c * 2^8 + d)
+ (a * 2^8 + b') + (c' * 2^8 + d)
= C + 2^8 * (a - a + c' - c) + (b' - b + d - d)
= C + 2^8 * (c' - c) + (b' - b)
= C - (2^8 * c + b) + (2^8 * c' + b')

So an unaligned incremental checksum can be done in a single operation
by byteswapping the changed bytes before passing them to the helper.
This patch implements that trick for unaligned MSS option updates.

Signed-off-by: Kacper Kokot <kacper.kokot.44@xxxxxxxxx>
---
Reproduction script

#!/usr/bin/env python3
import argparse
from scapy.all import *

parser = argparse.ArgumentParser()
parser.add_argument("target_ip")
parser.add_argument("target_port", type=int)
args = parser.parse_args()

def try_handshake(options):
sport = RandShort()
ip = IP(dst=args.target_ip)

syn = TCP(
sport=sport,
dport=args.target_port,
flags="S",
seq=1000,
options=options
)

synack = sr1(ip/syn, timeout=2)

print("SYNACK", synack)
if synack and synack.haslayer(TCP) and synack[TCP].flags == 0x12:
ack = TCP(
sport=sport,
dport=args.target_port,
flags="R",
seq=syn.seq + 1,
ack=synack.seq + 1,
)

send(ip/ack)
print("SYN-ACK received")
else:
print("No SYN-ACK received")

print("\n>>> MSS Aligned")
try_handshake([
('MSS', 1460),
("NOP", None),
("NOP", None),
('SAckOK', b''),
('Timestamp', (12345, 0)),
('WScale', 7)
])

print("\n>>> MSS Misaligned")
try_handshake([
("NOP", None),
('MSS', 1460),
("NOP", None),
('SAckOK', b''),
('Timestamp', (12345, 0)),
('WScale', 7)
])

A script to reproduce:

#!/usr/bin/env python3
import argparse
from scapy.all import *

parser = argparse.ArgumentParser()
parser.add_argument("target_ip")
parser.add_argument("target_port", type=int)
args = parser.parse_args()

mss_aligned_tcp_options = [
('MSS', 1460),
("NOP", None),
("NOP", None),
('SAckOK', b''),
('Timestamp', (12345, 0)),
('WScale', 7)
]

mss_misaligned_tcp_options = [
("NOP", None),
('MSS', 1460),
("NOP", None),
('SAckOK', b''),
('Timestamp', (12345, 0)),
('WScale', 7)
]

def try_handshake(options):
sport = RandShort()
ip = IP(dst=args.target_ip)

syn = TCP(
sport=sport,
dport=args.target_port,
flags="S",
seq=1000,
options=options
)

synack = sr1(ip/syn, timeout=2)

print("SYNACK", synack)

if synack and synack.haslayer(TCP) and synack[TCP].flags == 0x12:
ack = TCP(
sport=sport,
dport=args.target_port,
flags="R",
seq=syn.seq + 1,
ack=synack.seq + 1,
)

send(ip/ack)
print("SYN-ACK response")
else:
print("No SYN-ACK received")

print("\n>>> MSS Aligned")
try_handshake(mss_aligned_tcp_options)
print("\n>>> MSS Misaligned")
try_handshake(mss_misaligned_tcp_options)

net/netfilter/xt_TCPMSS.c | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/net/netfilter/xt_TCPMSS.c b/net/netfilter/xt_TCPMSS.c
index 80e1634bc51f..8e409858505d 100644
--- a/net/netfilter/xt_TCPMSS.c
+++ b/net/netfilter/xt_TCPMSS.c
@@ -117,6 +117,7 @@ tcpmss_mangle_packet(struct sk_buff *skb,
for (i = sizeof(struct tcphdr); i <= tcp_hdrlen - TCPOLEN_MSS; i += optlen(opt, i)) {
if (opt[i] == TCPOPT_MSS && opt[i+1] == TCPOLEN_MSS) {
u_int16_t oldmss;
+ u16 csum_oldmss, csum_newmss;

oldmss = (opt[i+2] << 8) | opt[i+3];

@@ -130,8 +131,19 @@ tcpmss_mangle_packet(struct sk_buff *skb,
opt[i+2] = (newmss & 0xff00) >> 8;
opt[i+3] = newmss & 0x00ff;

+ csum_oldmss = htons(oldmss);
+ csum_newmss = htons(newmss);
+
+ /* MSS may be unaligned; fix up the incremental checksum
+ * to avoid an invalid checksum and a dropped packet.
+ */
+ if (((char *)&opt[i + 2] - (char *)tcph) & 0x1 != 0) {
+ csum_oldmss = swab16(csum_oldmss);
+ csum_newmss = swab16(csum_newmss);
+ }
+
inet_proto_csum_replace2(&tcph->check, skb,
- htons(oldmss), htons(newmss),
+ csum_oldmss, csum_newmss,
false);
return 0;
}
--
2.43.0