Re: [PATCH] pipe_read: don't wake up the writer if the pipe is still full
From: Sapkal, Swapnil
Date: Fri Feb 28 2025 - 00:58:44 EST
Hi Oleg,
On 2/28/2025 2:42 AM, Oleg Nesterov wrote:
Sapkal, first of all, thanks again!
On 02/27, Sapkal, Swapnil wrote:
1. with 1 fd instead of 20:
/usr/bin/hackbench -g 16 -f 1 --threads --pipe -l 100000 -s 100
With this I was not able to reproduce the issue. I tried almost 5000
iterations.
OK,
2. with a size which divides 4096 evenly (e.g., 128):
...
When I retain the number of
groups to 16 and change the message size to 128, it took me around 150
iterations to reproduce this issue (with 100 bytes it was 20 iterations).
The exact command was
/usr/bin/hackbench -g 16 -f 20 --threads --pipe -l 100000 -s 128
Ah, good. This is good ;)
I will try to sprinkle some trace_printk's in the code where the state of
the pipe changes. I will report here if I find something.
Great! but...
Sapkal, I was going to finish (and test! ;) the patch below tomorrow, after
you test the previous debugging patch I sent in this thread. But since you
are going to change the kernel...
For the moment, please forget about that (as Mateusz pointed buggy) patch.
Could you apply the patch below and reproduce the problem ?
Yes, I was able to reproduce the problem with the below patch.
If yes, please do prctl(666) after the hang and send us the output from
dmesg, between "DUMP START" and "DUMP END". You can just do
$ perl -e 'syscall 157,666'
to call prctl(666) and trigger the dump.
I found a case in the dump where the pipe is empty still both reader and
writer are waiting on it.
[ 1397.829761] E=1 F=0; W=1719147 R=1719147
[ 1397.837843] RD=1 WR=1
Full dump is attached below.
Oleg.
---
diff --git a/fs/pipe.c b/fs/pipe.c
index b0641f75b1ba..566c75a0ff81 100644
--- a/fs/pipe.c
+++ b/fs/pipe.c
@@ -376,6 +376,8 @@ anon_pipe_read(struct kiocb *iocb, struct iov_iter *to)
}
if (pipe_empty(pipe->head, pipe->tail))
wake_next_reader = false;
+ if (ret > 0)
+ pipe->r_cnt++;
mutex_unlock(&pipe->mutex);
if (wake_writer)
@@ -565,6 +567,8 @@ anon_pipe_write(struct kiocb *iocb, struct iov_iter *from)
out:
if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
wake_next_writer = false;
+ if (ret > 0)
+ pipe->w_cnt++;
mutex_unlock(&pipe->mutex);
/*
@@ -695,6 +699,42 @@ pipe_poll(struct file *filp, poll_table *wait)
return mask;
}
+static DEFINE_MUTEX(PI_MUTEX);
+static LIST_HEAD(PI_LIST);
+
+void pi_dump(void);
+void pi_dump(void)
+{
+ struct pipe_inode_info *pipe;
+
+ pr_crit("---------- DUMP START ----------\n");
+ mutex_lock(&PI_MUTEX);
+ list_for_each_entry(pipe, &PI_LIST, pi_list) {
+ unsigned head, tail;
+
+ mutex_lock(&pipe->mutex);
+ head = pipe->head;
+ tail = pipe->tail;
+ pr_crit("E=%d F=%d; W=%d R=%d\n",
+ pipe_empty(head, tail), pipe_full(head, tail, pipe->max_usage),
+ pipe->w_cnt, pipe->r_cnt);
+
+// INCOMPLETE
+pr_crit("RD=%d WR=%d\n", waitqueue_active(&pipe->rd_wait), waitqueue_active(&pipe->wr_wait));
+
+ for (; tail < head; tail++) {
+ struct pipe_buffer *buf = pipe_buf(pipe, tail);
+ WARN_ON(buf->ops != &anon_pipe_buf_ops);
+ pr_crit("buf: o=%d l=%d\n", buf->offset, buf->len);
+ }
+ pr_crit("\n");
+
+ mutex_unlock(&pipe->mutex);
+ }
+ mutex_unlock(&PI_MUTEX);
+ pr_crit("---------- DUMP END ------------\n");
+}
+
static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)
{
int kill = 0;
@@ -706,8 +746,14 @@ static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)
}
spin_unlock(&inode->i_lock);
- if (kill)
+ if (kill) {
+ if (!list_empty(&pipe->pi_list)) {
+ mutex_lock(&PI_MUTEX);
+ list_del_init(&pipe->pi_list);
+ mutex_unlock(&PI_MUTEX);
+ }
free_pipe_info(pipe);
+ }
}
static int
@@ -790,6 +836,13 @@ struct pipe_inode_info *alloc_pipe_info(void)
if (pipe == NULL)
goto out_free_uid;
+ INIT_LIST_HEAD(&pipe->pi_list);
+ if (!strcmp(current->comm, "hackbench")) {
+ mutex_lock(&PI_MUTEX);
+ list_add_tail(&pipe->pi_list, &PI_LIST);
+ mutex_unlock(&PI_MUTEX);
+ }
+
if (pipe_bufs * PAGE_SIZE > max_size && !capable(CAP_SYS_RESOURCE))
pipe_bufs = max_size >> PAGE_SHIFT;
diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h
index 8ff23bf5a819..48d9bf5171dc 100644
--- a/include/linux/pipe_fs_i.h
+++ b/include/linux/pipe_fs_i.h
@@ -80,6 +80,9 @@ struct pipe_inode_info {
#ifdef CONFIG_WATCH_QUEUE
struct watch_queue *watch_queue;
#endif
+
+ struct list_head pi_list;
+ unsigned w_cnt, r_cnt;
};
/*
diff --git a/kernel/sys.c b/kernel/sys.c
index 4efca8a97d62..a85e34861b2e 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -2483,6 +2483,11 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
error = 0;
switch (option) {
+ case 666: {
+ extern void pi_dump(void);
+ pi_dump();
+ break;
+ }
case PR_SET_PDEATHSIG:
if (!valid_signal(arg2)) {
error = -EINVAL;
--
Thanks and Regards,
Swapnil[ 1394.383241] ---------- DUMP START ----------
[ 1394.388211] E=1 F=0; W=640 R=640
[ 1394.392001] RD=0 WR=0
[ 1394.396300] E=0 F=0; W=1 R=0
[ 1394.399625] RD=0 WR=0
[ 1394.402219] buf: o=0 l=1
[ 1394.406824] E=1 F=0; W=2000000 R=2000000
[ 1394.411322] RD=0 WR=0
[ 1394.415621] E=1 F=0; W=2000000 R=2000000
[ 1394.420108] RD=0 WR=0
[ 1394.424404] E=1 F=0; W=2000000 R=2000000
[ 1394.428898] RD=0 WR=0
[ 1394.433197] E=1 F=0; W=2000000 R=2000000
[ 1394.437692] RD=0 WR=0
[ 1394.441992] E=1 F=0; W=2000000 R=2000000
[ 1394.446490] RD=0 WR=0
[ 1394.450787] E=1 F=0; W=2000000 R=2000000
[ 1394.455282] RD=0 WR=0
[ 1394.459578] E=1 F=0; W=2000000 R=2000000
[ 1394.464073] RD=0 WR=0
[ 1394.468369] E=1 F=0; W=2000000 R=2000000
[ 1394.472866] RD=0 WR=0
[ 1394.477171] E=1 F=0; W=2000000 R=2000000
[ 1394.481666] RD=0 WR=0
[ 1394.485962] E=1 F=0; W=2000000 R=2000000
[ 1394.490460] RD=0 WR=0
[ 1394.494756] E=1 F=0; W=2000000 R=2000000
[ 1394.499252] RD=0 WR=0
[ 1394.503550] E=1 F=0; W=2000000 R=2000000
[ 1394.508044] RD=0 WR=0
[ 1394.512341] E=1 F=0; W=2000000 R=2000000
[ 1394.520567] RD=0 WR=0
[ 1394.532241] E=1 F=0; W=2000000 R=2000000
[ 1394.540298] RD=0 WR=0
[ 1394.551509] E=1 F=0; W=2000000 R=2000000
[ 1394.559501] RD=0 WR=0
[ 1394.570844] E=1 F=0; W=2000000 R=2000000
[ 1394.578883] RD=0 WR=0
[ 1394.590320] E=1 F=0; W=2000000 R=2000000
[ 1394.598359] RD=0 WR=0
[ 1394.609810] E=1 F=0; W=2000000 R=2000000
[ 1394.617850] RD=0 WR=0
[ 1394.629299] E=1 F=0; W=2000000 R=2000000
[ 1394.637334] RD=0 WR=0
[ 1394.648784] E=1 F=0; W=2000000 R=2000000
[ 1394.656837] RD=0 WR=0
[ 1394.668310] E=1 F=0; W=2000000 R=2000000
[ 1394.676365] RD=0 WR=0
[ 1394.687846] E=1 F=0; W=2000000 R=2000000
[ 1394.695906] RD=0 WR=0
[ 1394.707397] E=1 F=0; W=2000000 R=2000000
[ 1394.715461] RD=0 WR=0
[ 1394.726965] E=1 F=0; W=2000000 R=2000000
[ 1394.735036] RD=0 WR=0
[ 1394.746535] E=1 F=0; W=2000000 R=2000000
[ 1394.754609] RD=0 WR=0
[ 1394.766125] E=1 F=0; W=2000000 R=2000000
[ 1394.774200] RD=0 WR=0
[ 1394.785729] E=1 F=0; W=2000000 R=2000000
[ 1394.793813] RD=0 WR=0
[ 1394.805347] E=1 F=0; W=2000000 R=2000000
[ 1394.813433] RD=0 WR=0
[ 1394.824981] E=1 F=0; W=2000000 R=2000000
[ 1394.833074] RD=0 WR=0
[ 1394.844628] E=1 F=0; W=2000000 R=2000000
[ 1394.852720] RD=0 WR=0
[ 1394.864289] E=1 F=0; W=2000000 R=2000000
[ 1394.872388] RD=0 WR=0
[ 1394.883938] E=1 F=0; W=2000000 R=2000000
[ 1394.892030] RD=0 WR=0
[ 1394.903579] E=1 F=0; W=2000000 R=2000000
[ 1394.911673] RD=0 WR=0
[ 1394.923222] E=1 F=0; W=2000000 R=2000000
[ 1394.931321] RD=0 WR=0
[ 1394.942876] E=1 F=0; W=2000000 R=2000000
[ 1394.950972] RD=0 WR=0
[ 1394.962527] E=1 F=0; W=2000000 R=2000000
[ 1394.970623] RD=0 WR=0
[ 1394.982182] E=1 F=0; W=2000000 R=2000000
[ 1394.990279] RD=0 WR=0
[ 1395.001833] E=1 F=0; W=2000000 R=2000000
[ 1395.009942] RD=0 WR=0
[ 1395.021516] E=1 F=0; W=2000000 R=2000000
[ 1395.029623] RD=0 WR=0
[ 1395.041200] E=1 F=0; W=2000000 R=2000000
[ 1395.049310] RD=0 WR=0
[ 1395.060882] E=1 F=0; W=2000000 R=2000000
[ 1395.068986] RD=0 WR=0
[ 1395.080560] E=1 F=0; W=2000000 R=2000000
[ 1395.088667] RD=0 WR=0
[ 1395.100238] E=1 F=0; W=2000000 R=2000000
[ 1395.108348] RD=0 WR=0
[ 1395.119903] E=1 F=0; W=2000000 R=2000000
[ 1395.128002] RD=0 WR=0
[ 1395.139557] E=1 F=0; W=2000000 R=2000000
[ 1395.147646] RD=0 WR=0
[ 1395.159199] E=1 F=0; W=2000000 R=2000000
[ 1395.167295] RD=0 WR=0
[ 1395.178848] E=1 F=0; W=2000000 R=2000000
[ 1395.186942] RD=0 WR=0
[ 1395.198491] E=1 F=0; W=2000000 R=2000000
[ 1395.206587] RD=0 WR=0
[ 1395.218143] E=1 F=0; W=2000000 R=2000000
[ 1395.226241] RD=0 WR=0
[ 1395.237794] E=1 F=0; W=2000000 R=2000000
[ 1395.245893] RD=0 WR=0
[ 1395.257452] E=1 F=0; W=2000000 R=2000000
[ 1395.265548] RD=0 WR=0
[ 1395.277111] E=1 F=0; W=2000000 R=2000000
[ 1395.285212] RD=0 WR=0
[ 1395.296779] E=1 F=0; W=2000000 R=2000000
[ 1395.304873] RD=0 WR=0
[ 1395.316446] E=1 F=0; W=2000000 R=2000000
[ 1395.324541] RD=0 WR=0
[ 1395.336087] E=1 F=0; W=2000000 R=2000000
[ 1395.344163] RD=0 WR=0
[ 1395.355701] E=1 F=0; W=2000000 R=2000000
[ 1395.363787] RD=0 WR=0
[ 1395.375325] E=1 F=0; W=2000000 R=2000000
[ 1395.383409] RD=0 WR=0
[ 1395.394937] E=1 F=0; W=2000000 R=2000000
[ 1395.403015] RD=0 WR=0
[ 1395.414546] E=1 F=0; W=2000000 R=2000000
[ 1395.422630] RD=0 WR=0
[ 1395.434162] E=1 F=0; W=2000000 R=2000000
[ 1395.442248] RD=0 WR=0
[ 1395.453797] E=1 F=0; W=2000000 R=2000000
[ 1395.461886] RD=0 WR=0
[ 1395.473432] E=1 F=0; W=2000000 R=2000000
[ 1395.481526] RD=0 WR=0
[ 1395.493080] E=1 F=0; W=2000000 R=2000000
[ 1395.501175] RD=0 WR=0
[ 1395.512740] E=1 F=0; W=2000000 R=2000000
[ 1395.520835] RD=0 WR=0
[ 1395.532396] E=1 F=0; W=2000000 R=2000000
[ 1395.540497] RD=0 WR=0
[ 1395.552049] E=1 F=0; W=2000000 R=2000000
[ 1395.560149] RD=0 WR=0
[ 1395.571700] E=1 F=0; W=2000000 R=2000000
[ 1395.579792] RD=0 WR=0
[ 1395.591330] E=1 F=0; W=2000000 R=2000000
[ 1395.599416] RD=0 WR=0
[ 1395.610953] E=1 F=0; W=2000000 R=2000000
[ 1395.619041] RD=0 WR=0
[ 1395.630576] E=1 F=0; W=2000000 R=2000000
[ 1395.638666] RD=0 WR=0
[ 1395.650211] E=1 F=0; W=2000000 R=2000000
[ 1395.658295] RD=0 WR=0
[ 1395.669832] E=1 F=0; W=2000000 R=2000000
[ 1395.677915] RD=0 WR=0
[ 1395.689457] E=1 F=0; W=2000000 R=2000000
[ 1395.697543] RD=0 WR=0
[ 1395.709091] E=1 F=0; W=2000000 R=2000000
[ 1395.717172] RD=0 WR=0
[ 1395.728749] E=1 F=0; W=2000000 R=2000000
[ 1395.736846] RD=0 WR=0
[ 1395.748398] E=1 F=0; W=2000000 R=2000000
[ 1395.756488] RD=0 WR=0
[ 1395.768035] E=1 F=0; W=2000000 R=2000000
[ 1395.776117] RD=0 WR=0
[ 1395.787658] E=1 F=0; W=2000000 R=2000000
[ 1395.795746] RD=0 WR=0
[ 1395.807289] E=1 F=0; W=2000000 R=2000000
[ 1395.815363] RD=0 WR=0
[ 1395.826900] E=1 F=0; W=2000000 R=2000000
[ 1395.834983] RD=0 WR=0
[ 1395.846510] E=1 F=0; W=2000000 R=2000000
[ 1395.854584] RD=0 WR=0
[ 1395.866107] E=1 F=0; W=2000000 R=2000000
[ 1395.874203] RD=0 WR=0
[ 1395.885740] E=1 F=0; W=2000000 R=2000000
[ 1395.893826] RD=0 WR=0
[ 1395.905373] E=1 F=0; W=2000000 R=2000000
[ 1395.913469] RD=0 WR=0
[ 1395.925031] E=1 F=0; W=2000000 R=2000000
[ 1395.933115] RD=0 WR=0
[ 1395.944654] E=1 F=0; W=2000000 R=2000000
[ 1395.952751] RD=0 WR=0
[ 1395.964305] E=1 F=0; W=2000000 R=2000000
[ 1395.972399] RD=0 WR=0
[ 1395.983967] E=1 F=0; W=2000000 R=2000000
[ 1395.992063] RD=0 WR=0
[ 1396.003614] E=1 F=0; W=2000000 R=2000000
[ 1396.011715] RD=0 WR=0
[ 1396.023270] E=1 F=0; W=2000000 R=2000000
[ 1396.031359] RD=0 WR=0
[ 1396.042909] E=1 F=0; W=2000000 R=2000000
[ 1396.051001] RD=0 WR=0
[ 1396.062547] E=1 F=0; W=2000000 R=2000000
[ 1396.070642] RD=0 WR=0
[ 1396.082190] E=1 F=0; W=2000000 R=2000000
[ 1396.090283] RD=0 WR=0
[ 1396.101841] E=1 F=0; W=2000000 R=2000000
[ 1396.109930] RD=0 WR=0
[ 1396.121489] E=1 F=0; W=2000000 R=2000000
[ 1396.129588] RD=0 WR=0
[ 1396.141143] E=1 F=0; W=2000000 R=2000000
[ 1396.149239] RD=0 WR=0
[ 1396.160801] E=1 F=0; W=2000000 R=2000000
[ 1396.168902] RD=0 WR=0
[ 1396.180467] E=1 F=0; W=2000000 R=2000000
[ 1396.188569] RD=0 WR=0
[ 1396.200135] E=1 F=0; W=2000000 R=2000000
[ 1396.208238] RD=0 WR=0
[ 1396.219802] E=1 F=0; W=2000000 R=2000000
[ 1396.227898] RD=0 WR=0
[ 1396.239459] E=1 F=0; W=2000000 R=2000000
[ 1396.247559] RD=0 WR=0
[ 1396.259110] E=1 F=0; W=2000000 R=2000000
[ 1396.267204] RD=0 WR=0
[ 1396.278760] E=1 F=0; W=2000000 R=2000000
[ 1396.286859] RD=0 WR=0
[ 1396.298417] E=1 F=0; W=2000000 R=2000000
[ 1396.306514] RD=0 WR=0
[ 1396.318052] E=1 F=0; W=2000000 R=2000000
[ 1396.326137] RD=0 WR=0
[ 1396.337686] E=1 F=0; W=2000000 R=2000000
[ 1396.345771] RD=0 WR=0
[ 1396.357315] E=1 F=0; W=2000000 R=2000000
[ 1396.365403] RD=0 WR=0
[ 1396.376947] E=1 F=0; W=2000000 R=2000000
[ 1396.385032] RD=0 WR=0
[ 1396.396589] E=1 F=0; W=2000000 R=2000000
[ 1396.404677] RD=0 WR=0
[ 1396.416224] E=1 F=0; W=2000000 R=2000000
[ 1396.424312] RD=0 WR=0
[ 1396.435877] E=1 F=0; W=2000000 R=2000000
[ 1396.443971] RD=0 WR=0
[ 1396.455514] E=1 F=0; W=2000000 R=2000000
[ 1396.463602] RD=0 WR=0
[ 1396.475133] E=1 F=0; W=2000000 R=2000000
[ 1396.483219] RD=0 WR=0
[ 1396.494754] E=1 F=0; W=2000000 R=2000000
[ 1396.502841] RD=0 WR=0
[ 1396.514380] E=1 F=0; W=2000000 R=2000000
[ 1396.522466] RD=0 WR=0
[ 1396.533997] E=1 F=0; W=2000000 R=2000000
[ 1396.542082] RD=0 WR=0
[ 1396.553623] E=1 F=0; W=2000000 R=2000000
[ 1396.561716] RD=0 WR=0
[ 1396.573262] E=1 F=0; W=2000000 R=2000000
[ 1396.581351] RD=0 WR=0
[ 1396.592900] E=1 F=0; W=2000000 R=2000000
[ 1396.600991] RD=0 WR=0
[ 1396.612548] E=1 F=0; W=2000000 R=2000000
[ 1396.620643] RD=0 WR=0
[ 1396.632190] E=1 F=0; W=2000000 R=2000000
[ 1396.640275] RD=0 WR=0
[ 1396.651826] E=1 F=0; W=2000000 R=2000000
[ 1396.659919] RD=0 WR=0
[ 1396.671472] E=1 F=0; W=2000000 R=2000000
[ 1396.679572] RD=0 WR=0
[ 1396.691123] E=1 F=0; W=2000000 R=2000000
[ 1396.699220] RD=0 WR=0
[ 1396.710769] E=1 F=0; W=2000000 R=2000000
[ 1396.718862] RD=0 WR=0
[ 1396.730400] E=1 F=0; W=2000000 R=2000000
[ 1396.738487] RD=0 WR=0
[ 1396.750011] E=1 F=0; W=2000000 R=2000000
[ 1396.758092] RD=0 WR=0
[ 1396.769612] E=1 F=0; W=2000000 R=2000000
[ 1396.777695] RD=0 WR=0
[ 1396.789224] E=1 F=0; W=2000000 R=2000000
[ 1396.797305] RD=0 WR=0
[ 1396.808835] E=1 F=0; W=2000000 R=2000000
[ 1396.816914] RD=0 WR=0
[ 1396.828440] E=1 F=0; W=2000000 R=2000000
[ 1396.836522] RD=0 WR=0
[ 1396.848060] E=1 F=0; W=2000000 R=2000000
[ 1396.856140] RD=0 WR=0
[ 1396.867661] E=1 F=0; W=2000000 R=2000000
[ 1396.875742] RD=0 WR=0
[ 1396.887286] E=1 F=0; W=2000000 R=2000000
[ 1396.895367] RD=0 WR=0
[ 1396.906898] E=1 F=0; W=2000000 R=2000000
[ 1396.914978] RD=0 WR=0
[ 1396.926502] E=1 F=0; W=2000000 R=2000000
[ 1396.934587] RD=0 WR=0
[ 1396.946117] E=1 F=0; W=2000000 R=2000000
[ 1396.954214] RD=0 WR=0
[ 1396.965749] E=1 F=0; W=2000000 R=2000000
[ 1396.973834] RD=0 WR=0
[ 1396.985370] E=1 F=0; W=2000000 R=2000000
[ 1396.993458] RD=0 WR=0
[ 1397.005001] E=1 F=0; W=2000000 R=2000000
[ 1397.013098] RD=0 WR=0
[ 1397.024636] E=1 F=0; W=2000000 R=2000000
[ 1397.032727] RD=0 WR=0
[ 1397.044276] E=1 F=0; W=2000000 R=2000000
[ 1397.052374] RD=0 WR=0
[ 1397.063936] E=1 F=0; W=2000000 R=2000000
[ 1397.072036] RD=0 WR=0
[ 1397.083591] E=1 F=0; W=2000000 R=2000000
[ 1397.091689] RD=0 WR=0
[ 1397.103256] E=1 F=0; W=2000000 R=2000000
[ 1397.111359] RD=0 WR=0
[ 1397.122917] E=1 F=0; W=2000000 R=2000000
[ 1397.131013] RD=0 WR=0
[ 1397.142558] E=1 F=0; W=2000000 R=2000000
[ 1397.150649] RD=0 WR=0
[ 1397.162209] E=1 F=0; W=2000000 R=2000000
[ 1397.170301] RD=0 WR=0
[ 1397.181852] E=1 F=0; W=2000000 R=2000000
[ 1397.189943] RD=0 WR=0
[ 1397.201487] E=1 F=0; W=2000000 R=2000000
[ 1397.209580] RD=0 WR=0
[ 1397.221115] E=1 F=0; W=2000000 R=2000000
[ 1397.229204] RD=0 WR=0
[ 1397.240745] E=1 F=0; W=2000000 R=2000000
[ 1397.248839] RD=0 WR=0
[ 1397.260372] E=1 F=0; W=2000000 R=2000000
[ 1397.268463] RD=0 WR=0
[ 1397.280014] E=1 F=0; W=2000000 R=2000000
[ 1397.288103] RD=0 WR=0
[ 1397.299647] E=1 F=0; W=2000000 R=2000000
[ 1397.307734] RD=0 WR=0
[ 1397.319280] E=1 F=0; W=2000000 R=2000000
[ 1397.327370] RD=0 WR=0
[ 1397.338916] E=1 F=0; W=2000000 R=2000000
[ 1397.347001] RD=0 WR=0
[ 1397.358548] E=1 F=0; W=2000000 R=2000000
[ 1397.366632] RD=0 WR=0
[ 1397.378157] E=1 F=0; W=2000000 R=2000000
[ 1397.386246] RD=0 WR=0
[ 1397.397798] E=1 F=0; W=2000000 R=2000000
[ 1397.405888] RD=0 WR=0
[ 1397.417423] E=1 F=0; W=2000000 R=2000000
[ 1397.425510] RD=0 WR=0
[ 1397.437064] E=1 F=0; W=2000000 R=2000000
[ 1397.445155] RD=0 WR=0
[ 1397.456709] E=1 F=0; W=2000000 R=2000000
[ 1397.464800] RD=0 WR=0
[ 1397.476337] E=1 F=0; W=2000000 R=2000000
[ 1397.484430] RD=0 WR=0
[ 1397.495986] E=1 F=0; W=2000000 R=2000000
[ 1397.504081] RD=0 WR=0
[ 1397.515639] E=1 F=0; W=2000000 R=2000000
[ 1397.523737] RD=0 WR=0
[ 1397.535289] E=1 F=0; W=2000000 R=2000000
[ 1397.543372] RD=0 WR=0
[ 1397.554925] E=1 F=0; W=2000000 R=2000000
[ 1397.563021] RD=0 WR=0
[ 1397.574562] E=1 F=0; W=2000000 R=2000000
[ 1397.582654] RD=0 WR=0
[ 1397.594199] E=1 F=0; W=2000000 R=2000000
[ 1397.602287] RD=0 WR=0
[ 1397.613840] E=1 F=0; W=2000000 R=2000000
[ 1397.621934] RD=0 WR=0
[ 1397.633484] E=1 F=0; W=2000000 R=2000000
[ 1397.641570] RD=0 WR=0
[ 1397.653119] E=1 F=0; W=2000000 R=2000000
[ 1397.661211] RD=0 WR=0
[ 1397.672759] E=1 F=0; W=2000000 R=2000000
[ 1397.680850] RD=0 WR=0
[ 1397.692391] E=1 F=0; W=2000000 R=2000000
[ 1397.700474] RD=0 WR=0
[ 1397.712018] E=1 F=0; W=2000000 R=2000000
[ 1397.720110] RD=0 WR=0
[ 1397.731651] E=1 F=0; W=2000000 R=2000000
[ 1397.739739] RD=0 WR=0
[ 1397.751279] E=1 F=0; W=2000000 R=2000000
[ 1397.759370] RD=0 WR=0
[ 1397.770904] E=1 F=0; W=2000000 R=2000000
[ 1397.778992] RD=0 WR=0
[ 1397.790521] E=1 F=0; W=2000000 R=2000000
[ 1397.798607] RD=0 WR=0
[ 1397.810140] E=1 F=0; W=1719158 R=1719158
[ 1397.818226] RD=1 WR=0
[ 1397.829761] E=1 F=0; W=1719147 R=1719147
[ 1397.837843] RD=1 WR=1
[ 1397.849382] E=1 F=0; W=1719147 R=1719147
[ 1397.857465] RD=1 WR=0
[ 1397.869004] E=1 F=0; W=1719147 R=1719147
[ 1397.877090] RD=1 WR=0
[ 1397.888627] E=1 F=0; W=1719147 R=1719147
[ 1397.896715] RD=1 WR=0
[ 1397.908261] E=1 F=0; W=1719147 R=1719147
[ 1397.916348] RD=1 WR=0
[ 1397.927895] E=1 F=0; W=1719147 R=1719147
[ 1397.935981] RD=1 WR=0
[ 1397.947520] E=1 F=0; W=1719147 R=1719147
[ 1397.955600] RD=1 WR=0
[ 1397.967128] E=1 F=0; W=1719147 R=1719147
[ 1397.975211] RD=1 WR=0
[ 1397.986745] E=1 F=0; W=1719147 R=1719147
[ 1397.994830] RD=1 WR=0
[ 1398.006391] E=1 F=0; W=1719147 R=1719147
[ 1398.014480] RD=1 WR=0
[ 1398.026020] E=1 F=0; W=1719147 R=1719147
[ 1398.034104] RD=1 WR=0
[ 1398.045662] E=1 F=0; W=1719147 R=1719147
[ 1398.053754] RD=1 WR=0
[ 1398.065297] E=1 F=0; W=1719147 R=1719147
[ 1398.073385] RD=1 WR=0
[ 1398.084929] E=1 F=0; W=1719147 R=1719147
[ 1398.093022] RD=1 WR=0
[ 1398.104572] E=1 F=0; W=1719147 R=1719147
[ 1398.112666] RD=1 WR=0
[ 1398.124215] E=1 F=0; W=1719147 R=1719147
[ 1398.132303] RD=1 WR=0
[ 1398.143856] E=1 F=0; W=1719147 R=1719147
[ 1398.151945] RD=1 WR=0
[ 1398.163499] E=1 F=0; W=1719147 R=1719147
[ 1398.171592] RD=1 WR=0
[ 1398.183134] E=1 F=0; W=1719147 R=1719147
[ 1398.191225] RD=1 WR=0
[ 1398.202768] E=1 F=0; W=2000000 R=2000000
[ 1398.210859] RD=0 WR=0
[ 1398.222399] E=1 F=0; W=2000000 R=2000000
[ 1398.230485] RD=0 WR=0
[ 1398.242031] E=1 F=0; W=2000000 R=2000000
[ 1398.250122] RD=0 WR=0
[ 1398.261683] E=1 F=0; W=2000000 R=2000000
[ 1398.269786] RD=0 WR=0
[ 1398.281349] E=1 F=0; W=2000000 R=2000000
[ 1398.289447] RD=0 WR=0
[ 1398.301001] E=1 F=0; W=2000000 R=2000000
[ 1398.309100] RD=0 WR=0
[ 1398.320645] E=1 F=0; W=2000000 R=2000000
[ 1398.328735] RD=0 WR=0
[ 1398.340276] E=1 F=0; W=2000000 R=2000000
[ 1398.348363] RD=0 WR=0
[ 1398.359902] E=1 F=0; W=2000000 R=2000000
[ 1398.367988] RD=0 WR=0
[ 1398.379526] E=1 F=0; W=2000000 R=2000000
[ 1398.387609] RD=0 WR=0
[ 1398.399146] E=1 F=0; W=2000000 R=2000000
[ 1398.407231] RD=0 WR=0
[ 1398.418775] E=1 F=0; W=2000000 R=2000000
[ 1398.426858] RD=0 WR=0
[ 1398.438394] E=1 F=0; W=2000000 R=2000000
[ 1398.446475] RD=0 WR=0
[ 1398.458004] E=1 F=0; W=2000000 R=2000000
[ 1398.466084] RD=0 WR=0
[ 1398.477611] E=1 F=0; W=2000000 R=2000000
[ 1398.485695] RD=0 WR=0
[ 1398.497226] E=1 F=0; W=2000000 R=2000000
[ 1398.505308] RD=0 WR=0
[ 1398.516848] E=1 F=0; W=2000000 R=2000000
[ 1398.524934] RD=0 WR=0
[ 1398.536469] E=1 F=0; W=2000000 R=2000000
[ 1398.544554] RD=0 WR=0
[ 1398.556088] E=1 F=0; W=2000000 R=2000000
[ 1398.564171] RD=0 WR=0
[ 1398.575707] E=1 F=0; W=2000000 R=2000000
[ 1398.583794] RD=0 WR=0
[ 1398.595331] E=1 F=0; W=2000000 R=2000000
[ 1398.603421] RD=0 WR=0
[ 1398.614981] E=1 F=0; W=2000000 R=2000000
[ 1398.623073] RD=0 WR=0
[ 1398.634620] E=1 F=0; W=2000000 R=2000000
[ 1398.642716] RD=0 WR=0
[ 1398.654268] E=1 F=0; W=2000000 R=2000000
[ 1398.662367] RD=0 WR=0
[ 1398.673915] E=1 F=0; W=2000000 R=2000000
[ 1398.682011] RD=0 WR=0
[ 1398.693577] E=1 F=0; W=2000000 R=2000000
[ 1398.701673] RD=0 WR=0
[ 1398.713231] E=1 F=0; W=2000000 R=2000000
[ 1398.721325] RD=0 WR=0
[ 1398.732888] E=1 F=0; W=2000000 R=2000000
[ 1398.740983] RD=0 WR=0
[ 1398.752544] E=1 F=0; W=2000000 R=2000000
[ 1398.760640] RD=0 WR=0
[ 1398.772197] E=1 F=0; W=2000000 R=2000000
[ 1398.780294] RD=0 WR=0
[ 1398.791845] E=1 F=0; W=2000000 R=2000000
[ 1398.799943] RD=0 WR=0
[ 1398.811504] E=1 F=0; W=2000000 R=2000000
[ 1398.819602] RD=0 WR=0
[ 1398.831146] E=1 F=0; W=2000000 R=2000000
[ 1398.839237] RD=0 WR=0
[ 1398.850784] E=1 F=0; W=2000000 R=2000000
[ 1398.858874] RD=0 WR=0
[ 1398.870409] E=1 F=0; W=2000000 R=2000000
[ 1398.878494] RD=0 WR=0
[ 1398.890025] E=1 F=0; W=2000000 R=2000000
[ 1398.898108] RD=0 WR=0
[ 1398.909661] E=1 F=0; W=2000000 R=2000000
[ 1398.917746] RD=0 WR=0
[ 1398.929296] E=1 F=0; W=2000000 R=2000000
[ 1398.937384] RD=0 WR=0
[ 1398.948929] E=1 F=0; W=2000000 R=2000000
[ 1398.957019] RD=0 WR=0
[ 1398.968558] E=1 F=0; W=2000000 R=2000000
[ 1398.976647] RD=0 WR=0
[ 1398.988215] E=1 F=0; W=2000000 R=2000000
[ 1398.996319] RD=0 WR=0
[ 1399.007905] E=1 F=0; W=2000000 R=2000000
[ 1399.016001] RD=0 WR=0
[ 1399.027563] E=1 F=0; W=2000000 R=2000000
[ 1399.035657] RD=0 WR=0
[ 1399.047213] E=1 F=0; W=2000000 R=2000000
[ 1399.055307] RD=0 WR=0
[ 1399.066876] E=1 F=0; W=2000000 R=2000000
[ 1399.074979] RD=0 WR=0
[ 1399.086548] E=1 F=0; W=2000000 R=2000000
[ 1399.094652] RD=0 WR=0
[ 1399.106229] E=1 F=0; W=2000000 R=2000000
[ 1399.114341] RD=0 WR=0
[ 1399.125919] E=1 F=0; W=2000000 R=2000000
[ 1399.134026] RD=0 WR=0
[ 1399.145616] E=1 F=0; W=2000000 R=2000000
[ 1399.153720] RD=0 WR=0
[ 1399.165307] E=1 F=0; W=2000000 R=2000000
[ 1399.173423] RD=0 WR=0
[ 1399.185007] E=1 F=0; W=2000000 R=2000000
[ 1399.193113] RD=0 WR=0
[ 1399.204691] E=1 F=0; W=2000000 R=2000000
[ 1399.212812] RD=0 WR=0
[ 1399.224410] E=1 F=0; W=2000000 R=2000000
[ 1399.232525] RD=0 WR=0
[ 1399.244127] E=1 F=0; W=2000000 R=2000000
[ 1399.252246] RD=0 WR=0
[ 1399.263850] E=1 F=0; W=2000000 R=2000000
[ 1399.271971] RD=0 WR=0
[ 1399.283579] E=1 F=0; W=2000000 R=2000000
[ 1399.291701] RD=0 WR=0
[ 1399.303311] E=1 F=0; W=2000000 R=2000000
[ 1399.311441] RD=0 WR=0
[ 1399.323047] E=1 F=0; W=2000000 R=2000000
[ 1399.331165] RD=0 WR=0
[ 1399.342769] E=1 F=0; W=2000000 R=2000000
[ 1399.350887] RD=0 WR=0
[ 1399.362486] E=1 F=0; W=2000000 R=2000000
[ 1399.370593] RD=0 WR=0
[ 1399.382178] E=1 F=0; W=2000000 R=2000000
[ 1399.390291] RD=0 WR=0
[ 1399.401866] E=1 F=0; W=2000000 R=2000000
[ 1399.409971] RD=0 WR=0
[ 1399.421555] E=1 F=0; W=2000000 R=2000000
[ 1399.429669] RD=0 WR=0
[ 1399.441249] E=1 F=0; W=2000000 R=2000000
[ 1399.449361] RD=0 WR=0
[ 1399.460949] E=1 F=0; W=2000000 R=2000000
[ 1399.469068] RD=0 WR=0
[ 1399.480660] E=1 F=0; W=2000000 R=2000000
[ 1399.488778] RD=0 WR=0
[ 1399.500366] E=1 F=0; W=2000000 R=2000000
[ 1399.508475] RD=0 WR=0
[ 1399.520060] E=1 F=0; W=2000000 R=2000000
[ 1399.528170] RD=0 WR=0
[ 1399.539747] E=1 F=0; W=2000000 R=2000000
[ 1399.547850] RD=0 WR=0
[ 1399.559425] E=1 F=0; W=2000000 R=2000000
[ 1399.567530] RD=0 WR=0
[ 1399.579103] E=1 F=0; W=2000000 R=2000000
[ 1399.587207] RD=0 WR=0
[ 1399.598769] E=1 F=0; W=2000000 R=2000000
[ 1399.606865] RD=0 WR=0
[ 1399.618424] E=1 F=0; W=2000000 R=2000000
[ 1399.626520] RD=0 WR=0
[ 1399.638080] E=1 F=0; W=2000000 R=2000000
[ 1399.646175] RD=0 WR=0
[ 1399.657744] E=1 F=0; W=2000000 R=2000000
[ 1399.665842] RD=0 WR=0
[ 1399.677415] E=1 F=0; W=2000000 R=2000000
[ 1399.685518] RD=0 WR=0
[ 1399.697092] E=1 F=0; W=2000000 R=2000000
[ 1399.705197] RD=0 WR=0
[ 1399.716762] E=1 F=0; W=2000000 R=2000000
[ 1399.724865] RD=0 WR=0
[ 1399.736434] E=1 F=0; W=2000000 R=2000000
[ 1399.744534] RD=0 WR=0
[ 1399.756089] E=1 F=0; W=2000000 R=2000000
[ 1399.764183] RD=0 WR=0
[ 1399.775731] E=1 F=0; W=2000000 R=2000000
[ 1399.783818] RD=0 WR=0
[ 1399.795374] E=1 F=0; W=2000000 R=2000000
[ 1399.803458] RD=0 WR=0
[ 1399.814992] E=1 F=0; W=2000000 R=2000000
[ 1399.823076] RD=0 WR=0
[ 1399.834602] E=1 F=0; W=2000000 R=2000000
[ 1399.842684] RD=0 WR=0
[ 1399.854209] E=1 F=0; W=2000000 R=2000000
[ 1399.862290] RD=0 WR=0
[ 1399.873796] E=1 F=0; W=2000000 R=2000000
[ 1399.881873] RD=0 WR=0
[ 1399.893400] E=1 F=0; W=2000000 R=2000000
[ 1399.901480] RD=0 WR=0
[ 1399.912997] E=1 F=0; W=2000000 R=2000000
[ 1399.921074] RD=0 WR=0
[ 1399.932601] E=1 F=0; W=2000000 R=2000000
[ 1399.940682] RD=0 WR=0
[ 1399.952202] E=1 F=0; W=2000000 R=2000000
[ 1399.960281] RD=0 WR=0
[ 1399.971803] E=1 F=0; W=2000000 R=2000000
[ 1399.979874] RD=0 WR=0
[ 1399.991384] E=1 F=0; W=2000000 R=2000000
[ 1399.999464] RD=0 WR=0
[ 1400.011005] E=1 F=0; W=2000000 R=2000000
[ 1400.019094] RD=0 WR=0
[ 1400.030619] E=1 F=0; W=2000000 R=2000000
[ 1400.038705] RD=0 WR=0
[ 1400.050241] E=1 F=0; W=2000000 R=2000000
[ 1400.058336] RD=0 WR=0
[ 1400.069874] E=1 F=0; W=2000000 R=2000000
[ 1400.077959] RD=0 WR=0
[ 1400.089508] E=1 F=0; W=2000000 R=2000000
[ 1400.097597] RD=0 WR=0
[ 1400.109149] E=1 F=0; W=2000000 R=2000000
[ 1400.117240] RD=0 WR=0
[ 1400.128796] E=1 F=0; W=2000000 R=2000000
[ 1400.136891] RD=0 WR=0
[ 1400.148448] E=1 F=0; W=2000000 R=2000000
[ 1400.156539] RD=0 WR=0
[ 1400.168097] E=1 F=0; W=2000000 R=2000000
[ 1400.176193] RD=0 WR=0
[ 1400.187752] E=1 F=0; W=2000000 R=2000000
[ 1400.195848] RD=0 WR=0
[ 1400.207409] E=1 F=0; W=2000000 R=2000000
[ 1400.215509] RD=0 WR=0
[ 1400.227064] E=1 F=0; W=2000000 R=2000000
[ 1400.235158] RD=0 WR=0
[ 1400.246716] E=1 F=0; W=2000000 R=2000000
[ 1400.254812] RD=0 WR=0
[ 1400.266368] E=1 F=0; W=2000000 R=2000000
[ 1400.274461] RD=0 WR=0
[ 1400.286002] E=1 F=0; W=2000000 R=2000000
[ 1400.294091] RD=0 WR=0
[ 1400.305629] E=1 F=0; W=2000000 R=2000000
[ 1400.313722] RD=0 WR=0
[ 1400.325237] E=1 F=0; W=2000000 R=2000000
[ 1400.333316] RD=0 WR=0
[ 1400.344834] E=1 F=0; W=2000000 R=2000000
[ 1400.352912] RD=0 WR=0
[ 1400.364432] E=1 F=0; W=2000000 R=2000000
[ 1400.372514] RD=0 WR=0
[ 1400.384052] E=1 F=0; W=2000000 R=2000000
[ 1400.392134] RD=0 WR=0
[ 1400.403665] E=1 F=0; W=2000000 R=2000000
[ 1400.411755] RD=0 WR=0
[ 1400.423282] E=1 F=0; W=2000000 R=2000000
[ 1400.431365] RD=0 WR=0
[ 1400.442897] E=1 F=0; W=2000000 R=2000000
[ 1400.450984] RD=0 WR=0
[ 1400.462520] E=1 F=0; W=2000000 R=2000000
[ 1400.470607] RD=0 WR=0
[ 1400.482141] E=1 F=0; W=2000000 R=2000000
[ 1400.490225] RD=0 WR=0
[ 1400.501793] E=1 F=0; W=2000000 R=2000000
[ 1400.509886] RD=0 WR=0
[ 1400.521428] E=1 F=0; W=2000000 R=2000000
[ 1400.529517] RD=0 WR=0
[ 1400.541063] E=1 F=0; W=2000000 R=2000000
[ 1400.549149] RD=0 WR=0
[ 1400.560688] ---------- DUMP END ------------