diff --git a/include/net/tcp.h b/include/net/tcp.h
index 7de4ea3..c8fd034 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -272,6 +272,8 @@ DECLARE_SNMP_STAT(struct tcp_mib, tcp_statistics);
#define TCP_ADD_STATS_BH(field, val) SNMP_ADD_STATS_BH(tcp_statistics, field, val)
#define TCP_ADD_STATS_USER(field, val) SNMP_ADD_STATS_USER(tcp_statistics, field, val)
+extern void tcp_verify_lost(struct sock *sk);
+
extern void tcp_v4_err(struct sk_buff *skb, u32);
extern void tcp_shutdown (struct sock *sk, int how);
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index 19c449f..681ab1c 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -1604,6 +1604,7 @@ advance_sp:
tcp_mark_lost_retrans(sk);
tcp_verify_left_out(tp);
+ tcp_verify_lost(sk);
if ((reord < tp->fackets_out) &&
((icsk->icsk_ca_state != TCP_CA_Loss) || tp->undo_marker) &&
@@ -1647,6 +1648,7 @@ static void tcp_add_reno_sack(struct sock *sk)
tp->sacked_out++;
tcp_check_reno_reordering(sk, 0);
tcp_verify_left_out(tp);
+ tcp_verify_lost(sk);
}
/* Account for ACK, ACKing some data in Reno Recovery phase. */
@@ -1664,6 +1666,7 @@ static void tcp_remove_reno_sacks(struct sock *sk, int acked)
}
tcp_check_reno_reordering(sk, acked);
tcp_verify_left_out(tp);
+ tcp_verify_lost(sk);
}
static inline void tcp_reset_reno_sack(struct tcp_sock *tp)
@@ -1765,6 +1768,7 @@ void tcp_enter_frto(struct sock *sk)
tp->retrans_out -= tcp_skb_pcount(skb);
}
tcp_verify_left_out(tp);
+ tcp_verify_lost(sk);
/* Too bad if TCP was application limited */
tp->snd_cwnd = min(tp->snd_cwnd, tcp_packets_in_flight(tp) + 1);
@@ -1827,6 +1831,7 @@ static void tcp_enter_frto_loss(struct sock *sk, int allowed_segments, int flag)
}
}
tcp_verify_left_out(tp);
+ tcp_verify_lost(sk);
tp->snd_cwnd = tcp_packets_in_flight(tp) + allowed_segments;
tp->snd_cwnd_cnt = 0;
@@ -1912,6 +1917,7 @@ void tcp_enter_loss(struct sock *sk, int how)
}
}
tcp_verify_left_out(tp);
+ tcp_verify_lost(sk);
tp->reordering = min_t(unsigned int, tp->reordering,
sysctl_tcp_reordering);
@@ -2167,6 +2173,7 @@ static void tcp_mark_head_lost(struct sock *sk, int packets, int fast_rexmit)
}
}
tcp_verify_left_out(tp);
+ tcp_verify_lost(sk);
}
/* Account newly detected lost packet(s) */
@@ -2216,6 +2223,7 @@ static void tcp_update_scoreboard(struct sock *sk, int fast_rexmit)
tp->scoreboard_skb_hint = skb;
tcp_verify_left_out(tp);
+ tcp_verify_lost(sk);
}
}
@@ -2431,6 +2439,7 @@ static void tcp_try_to_open(struct sock *sk, int flag)
struct tcp_sock *tp = tcp_sk(sk);
tcp_verify_left_out(tp);
+ tcp_verify_lost(sk);
if (tp->retrans_out == 0)
tp->retrans_stamp = 0;
@@ -2526,6 +2535,7 @@ static void tcp_fastretrans_alert(struct sock *sk, int pkts_acked, int flag)
/* D. Check consistency of the current state. */
tcp_verify_left_out(tp);
+ tcp_verify_lost(sk);
/* E. Check state exit conditions. State can be terminated
* when high_seq is ACKed. */
@@ -2645,6 +2655,12 @@ static void tcp_fastretrans_alert(struct sock *sk, int pkts_acked, int flag)
if (do_lost || (tcp_is_fack(tp) && tcp_head_timedout(sk)))
tcp_update_scoreboard(sk, fast_rexmit);
tcp_cwnd_down(sk, flag);
+
+ if (WARN_ON(tcp_write_queue_head(sk) == NULL))
+ return;
+ if (WARN_ON(!tp->packets_out))
+ return;
+
tcp_xmit_retransmit_queue(sk);
}
@@ -2847,6 +2863,8 @@ static int tcp_clean_rtx_queue(struct sock *sk, int prior_fackets)
sk_wmem_free_skb(sk, skb);
tcp_clear_all_retrans_hints(tp);
}
+
+ tcp_verify_lost(sk);
if (skb && (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED))
flag |= FLAG_SACK_RENEGING;
@@ -2890,6 +2908,8 @@ static int tcp_clean_rtx_queue(struct sock *sk, int prior_fackets)
#if FASTRETRANS_DEBUG > 0
BUG_TRAP((int)tp->sacked_out >= 0);
BUG_TRAP((int)tp->lost_out >= 0);
+ if (tp->lost_out > tp->packets_out)
+ printk(KERN_ERR "Lost underflowed to %u\n", tp->lost_out);
BUG_TRAP((int)tp->retrans_out >= 0);
if (!tp->packets_out && tcp_is_sack(tp)) {
icsk = inet_csk(sk);
@@ -3061,6 +3081,7 @@ static int tcp_process_frto(struct sock *sk, int flag)
struct tcp_sock *tp = tcp_sk(sk);
tcp_verify_left_out(tp);
+ tcp_verify_lost(sk);
/* Duplicate the behavior from Loss state (fastretrans_alert) */
if (flag & FLAG_DATA_ACKED)
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 00156bf..410aada 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -108,6 +108,25 @@ struct inet_hashinfo __cacheline_aligned tcp_hashinfo = {
.lhash_wait = __WAIT_QUEUE_HEAD_INITIALIZER(tcp_hashinfo.lhash_wait),
};
+void tcp_verify_lost(struct sock *sk)
+{
+ struct tcp_sock *tp = tcp_sk(sk);
+ u32 lost = 0;
+ struct sk_buff *skb;
+
+ tcp_for_write_queue(skb, sk) {
+ if (skb == tcp_send_head(sk))
+ break;
+ if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST)
+ lost += tcp_skb_pcount(skb);
+ }
+
+ if (WARN_ON(lost != tp->lost_out)) {
+ printk(KERN_ERR "Lost: %u vs %u, %u (%d)\n", lost, tp->lost_out,
+ tp->packets_out, tcp_is_sack(tp));
+ }
+}
+
static inline __u32 tcp_v4_init_sequence(struct sk_buff *skb)
{
return secure_tcp_sequence_number(ip_hdr(skb)->daddr,
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index ed750f9..586660a 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -790,6 +790,8 @@ int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len,
skb_header_release(buff);
tcp_insert_write_queue_after(skb, buff, sk);
+ tcp_verify_lost(sk);
+
return 0;
}
@@ -1459,10 +1461,12 @@ static int tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle)
/* Do MTU probing. */
if ((result = tcp_mtu_probe(sk)) == 0) {
+ tcp_verify_lost(sk);
return 0;
} else if (result > 0) {
sent_pkts = 1;
}
+ tcp_verify_lost(sk);
while ((skb = tcp_send_head(sk))) {
unsigned int limit;
@@ -1764,6 +1768,7 @@ static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *skb,
tcp_clear_retrans_hints_partial(tp);
sk_wmem_free_skb(sk, next_skb);
+ tcp_verify_lost(sk);
}
/* Do a simple retransmit without using the backoff mechanisms in
@@ -1794,6 +1799,8 @@ void tcp_simple_retransmit(struct sock *sk)
}
}
}
+
+ tcp_verify_lost(sk);
tcp_clear_all_retrans_hints(tp);
@@ -1815,6 +1822,8 @@ void tcp_simple_retransmit(struct sock *sk)
tcp_set_ca_state(sk, TCP_CA_Loss);
}
tcp_xmit_retransmit_queue(sk);
+
+ tcp_verify_lost(sk);
}
/* This retransmits one SKB. Policy decisions and retransmit queue
@@ -1996,6 +2005,8 @@ void tcp_xmit_retransmit_queue(struct sock *sk)
}
}
}
+
+ tcp_verify_lost(sk);
/* OK, demanded retransmission is finished. */
@@ -2054,6 +2065,8 @@ void tcp_xmit_retransmit_queue(struct sock *sk)
NET_INC_STATS_BH(LINUX_MIB_TCPFORWARDRETRANS);
}
+
+ tcp_verify_lost(sk);
}
/* Send a fin. The caller locks the socket for us. This cannot be