[PATCH 2/3] netfilter : 3 patches to boost ip_tables performance

[Eric Dumazet]

Patch 2/3 (please apply after Patch 1/3)

2) Loop unrolling

It seems that with current compilers and CFLAGS, the code from
ip_packet_match() is very bad, using lot of mispredicted conditional branches 
I made some patches and generated code on i386 and x86_64
is much better.

Signed-off-by: Eric Dumazet <dada1@xxxxxxxxxxxxx>


--- linux-2.6/net/ipv4/netfilter/ip_tables.c	2005-09-21 23:55:30.000000000 +0200
+++ linux-2.6-ed/net/ipv4/netfilter/ip_tables.c	2005-09-22 00:39:29.000000000 +0200
@@ -125,6 +125,27 @@
 #define up(x) do { printk("UP:%u:" #x "\n", __LINE__); up(x); } while(0)
 #endif
 
+/*
+ * Special macro to compare IFNAMSIZ 'strings', with mask.
+ * Best results if feeded with 3 args pointing to 'unsigned long' types
+ * Loop is manually unrolled for performance reasons.
+ * gcc generates code without branches, IFNAMSIZ being a constant
+ */
+#define compare_if_lstrings(ret, devname, expect, expect_mask)			\
+	ret = ((devname)[0] ^ (expect)[0]) & (expect_mask)[0];			\
+	if (IFNAMSIZ > sizeof(*devname))					\
+		ret |= ((devname)[1] ^ (expect)[1]) & (expect_mask)[1];		\
+	if (IFNAMSIZ > 2 * sizeof(*devname))					\
+		ret |= ((devname)[2] ^ (expect)[2]) & (expect_mask)[2];		\
+	if (IFNAMSIZ > 3 * sizeof(*devname))					\
+		ret |= ((devname)[3] ^ (expect)[3]) & (expect_mask)[3];		\
+	/* just in case IFNAMSIZ is enlarged */					\
+	if (IFNAMSIZ > 4 * sizeof(*devname)) {					\
+		int i;								\
+		for (i = 4 ; (i < IFNAMSIZ/sizeof(*devname)); i++)		\
+			ret |= ((devname)[i] ^ (expect)[i]) & (expect_mask)[i];	\
+	}
+
 /* Returns whether matches rule or not. */
 static inline int
 ip_packet_match(const struct iphdr *ip,
@@ -133,16 +154,22 @@
 		const struct ipt_ip *ipinfo,
 		int isfrag)
 {
-	size_t i;
+	int bool1, bool2;
 	unsigned long ret;
 
 #define FWINV(bool,invflg) ((bool) ^ !!(ipinfo->invflags & invflg))
 
-	if (FWINV((ip->saddr&ipinfo->smsk.s_addr) != ipinfo->src.s_addr,
-		  IPT_INV_SRCIP)
-	    || FWINV((ip->daddr&ipinfo->dmsk.s_addr) != ipinfo->dst.s_addr,
-		     IPT_INV_DSTIP)) {
-		dprintf("Source or dest mismatch.\n");
+	bool1 = ((ip->saddr&ipinfo->smsk.s_addr) != ipinfo->src.s_addr);
+	bool1 ^= !!(ipinfo->invflags & IPT_INV_SRCIP);
+
+	bool2 = ((ip->daddr&ipinfo->dmsk.s_addr) != ipinfo->dst.s_addr);
+	bool2 ^= !!(ipinfo->invflags & IPT_INV_DSTIP);
+
+	if ((bool1 | bool2) != 0) {
+		if (bool1)
+			dprintf("Source%s mismatch.\n", bool2 ? " and Dest":"");
+		else
+			dprintf("Dest mismatch.\n");
 
 		dprintf("SRC: %u.%u.%u.%u. Mask: %u.%u.%u.%u. Target: %u.%u.%u.%u.%s\n",
 			NIPQUAD(ip->saddr),
@@ -157,27 +184,26 @@
 		return 0;
 	}
 
-	/* Look for ifname matches; this should unroll nicely. */
-	for (i = 0, ret = 0; i < IFNAMSIZ/sizeof(unsigned long); i++) {
-		ret |= (((const unsigned long *)indev)[i]
-			^ ((const unsigned long *)ipinfo->iniface)[i])
-			& ((const unsigned long *)ipinfo->iniface_mask)[i];
-	}
+	compare_if_lstrings(ret,
+		(const unsigned long *)indev,
+		(const unsigned long *)ipinfo->iniface,
+		(const unsigned long *)ipinfo->iniface_mask);
 
-	if (FWINV(ret != 0, IPT_INV_VIA_IN)) {
+	bool1 = FWINV(ret != 0, IPT_INV_VIA_IN);
+	if (bool1) {
 		dprintf("VIA in mismatch (%s vs %s).%s\n",
 			indev, ipinfo->iniface,
 			ipinfo->invflags&IPT_INV_VIA_IN ?" (INV)":"");
 		return 0;
 	}
 
-	for (i = 0, ret = 0; i < IFNAMSIZ/sizeof(unsigned long); i++) {
-		ret |= (((const unsigned long *)outdev)[i]
-			^ ((const unsigned long *)ipinfo->outiface)[i])
-			& ((const unsigned long *)ipinfo->outiface_mask)[i];
-	}
+	compare_if_lstrings(ret,
+		(const unsigned long *)outdev,
+		(const unsigned long *)ipinfo->outiface,
+		(const unsigned long *)ipinfo->outiface_mask);
 
-	if (FWINV(ret != 0, IPT_INV_VIA_OUT)) {
+	bool1 = FWINV(ret != 0, IPT_INV_VIA_OUT);
+	if (bool1) {
 		dprintf("VIA out mismatch (%s vs %s).%s\n",
 			outdev, ipinfo->outiface,
 			ipinfo->invflags&IPT_INV_VIA_OUT ?" (INV)":"");
@@ -185,8 +211,9 @@
 	}
 
 	/* Check specific protocol */
-	if (ipinfo->proto
-	    && FWINV(ip->protocol != ipinfo->proto, IPT_INV_PROTO)) {
+	bool1 = FWINV(ip->protocol != ipinfo->proto, IPT_INV_PROTO) ;
+	bool1 &= (ipinfo->proto != 0);
+	if (bool1) {
 		dprintf("Packet protocol %hi does not match %hi.%s\n",
 			ip->protocol, ipinfo->proto,
 			ipinfo->invflags&IPT_INV_PROTO ? " (INV)":"");