[lkp] [RAID5] 6ab2a4b806: 91.9% fsmark.app_overhead
From: kernel test robot
Date: Thu Mar 10 2016 - 21:03:49 EST
FYI, we noticed the below changes on
https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git master
commit 6ab2a4b806ae21b6c3e47c5ff1285ec06d505325 ("RAID5: revert e9e4c377e2f563 to fix a livelock")
=========================================================================================
compiler/cpufreq_governor/disk/filesize/fs/iterations/kconfig/md/nr_threads/rootfs/sync_method/tbox_group/test_size/testcase:
gcc-4.9/performance/8BRD_12G/4M/xfs/1x/x86_64-rhel/RAID5/64t/debian-x86_64-2015-02-07.cgz/fsyncBeforeClose/lkp-hsx02/60G/fsmark
commit:
27a353c026a879a1001e5eac4bda75b16262c44a
6ab2a4b806ae21b6c3e47c5ff1285ec06d505325
27a353c026a879a1 6ab2a4b806ae21b6c3e47c5ff1
---------------- --------------------------
%stddev %change %stddev
\ | \
592411 ± 2% +91.9% 1136931 ± 11% fsmark.app_overhead
98.37 ± 1% -21.9% 76.80 ± 2% fsmark.files_per_sec
188.95 ± 0% +59.9% 302.16 ± 3% fsmark.time.elapsed_time
188.95 ± 0% +59.9% 302.16 ± 3% fsmark.time.elapsed_time.max
20813 ± 4% +290.1% 81196 ± 4% fsmark.time.minor_page_faults
165.00 ± 2% +553.8% 1078 ± 7% fsmark.time.percent_of_cpu_this_job_got
312.06 ± 3% +947.5% 3269 ± 10% fsmark.time.system_time
10132990 ± 2% +465.7% 57325950 ± 7% fsmark.time.voluntary_context_switches
171460 ± 2% -24.4% 129549 ± 6% meminfo.Writeback
830948 ± 85% +339.1% 3649060 ± 89% numa-numastat.node3.numa_foreign
2814 ± 1% -10.4% 2523 ± 0% slabinfo.kmalloc-4096.active_objs
2870 ± 1% -12.1% 2523 ± 0% slabinfo.kmalloc-4096.num_objs
285.82 ± 0% +39.4% 398.30 ± 2% uptime.boot
40413 ± 0% +33.4% 53898 ± 2% uptime.idle
1.92 ± 1% +345.0% 8.53 ± 6% turbostat.%Busy
55.25 ± 1% +347.1% 247.00 ± 6% turbostat.Avg_MHz
356.24 ± 0% +5.5% 375.77 ± 0% turbostat.PkgWatt
7994 ± 3% +26.5% 10115 ± 5% softirqs.NET_RX
308842 ± 48% +222.4% 995686 ± 12% softirqs.RCU
227016 ± 26% +477.0% 1309870 ± 16% softirqs.SCHED
743051 ± 31% +195.6% 2196488 ± 8% softirqs.TIMER
329656 ± 0% -37.0% 207541 ± 3% vmstat.io.bo
1.75 ± 24% +500.0% 10.50 ± 4% vmstat.procs.r
108684 ± 1% +249.1% 379378 ± 3% vmstat.system.cs
3836 ± 2% +257.4% 13711 ± 5% vmstat.system.in
188.95 ± 0% +59.9% 302.16 ± 3% time.elapsed_time
188.95 ± 0% +59.9% 302.16 ± 3% time.elapsed_time.max
20813 ± 4% +290.1% 81196 ± 4% time.minor_page_faults
165.00 ± 2% +553.8% 1078 ± 7% time.percent_of_cpu_this_job_got
312.06 ± 3% +947.5% 3269 ± 10% time.system_time
10132990 ± 2% +465.7% 57325950 ± 7% time.voluntary_context_switches
2497 ±142% -87.3% 316.75 ± 15% numa-meminfo.node1.Inactive(anon)
2591 ±137% -84.5% 401.50 ± 11% numa-meminfo.node1.Shmem
16695 ± 1% -16.9% 13875 ± 9% numa-meminfo.node1.Unevictable
16915 ± 1% -17.4% 13969 ± 8% numa-meminfo.node2.Unevictable
5698 ± 22% +90.1% 10831 ± 24% numa-meminfo.node3.Active(anon)
5660 ± 22% +89.2% 10707 ± 24% numa-meminfo.node3.AnonPages
550.25 ± 22% +1135.0% 6795 ± 50% numa-meminfo.node3.Inactive(anon)
626.25 ± 19% +1002.1% 6902 ± 50% numa-meminfo.node3.Shmem
383.75 ± 3% +11.7% 428.75 ± 2% numa-vmstat.node1.nr_alloc_batch
623.75 ±142% -87.4% 78.75 ± 16% numa-vmstat.node1.nr_inactive_anon
647.25 ±137% -84.6% 100.00 ± 11% numa-vmstat.node1.nr_shmem
4173 ± 1% -16.9% 3468 ± 9% numa-vmstat.node1.nr_unevictable
4228 ± 1% -17.4% 3492 ± 8% numa-vmstat.node2.nr_unevictable
1424 ± 22% +90.1% 2708 ± 24% numa-vmstat.node3.nr_active_anon
1415 ± 22% +89.2% 2677 ± 24% numa-vmstat.node3.nr_anon_pages
137.00 ± 22% +1139.8% 1698 ± 50% numa-vmstat.node3.nr_inactive_anon
156.00 ± 19% +1005.8% 1725 ± 50% numa-vmstat.node3.nr_shmem
36.67 ±130% +52543.9% 19302 ±139% proc-vmstat.kswapd_high_wmark_hit_quickly
3.75 ± 11% +6480.0% 246.75 ±101% proc-vmstat.nr_pages_scanned
42860 ± 2% -24.4% 32415 ± 6% proc-vmstat.nr_writeback
10144 ± 8% +593.7% 70377 ± 4% proc-vmstat.numa_hint_faults
5990 ± 14% +608.3% 42426 ± 4% proc-vmstat.numa_hint_faults_local
1310 ± 14% +365.9% 6105 ± 11% proc-vmstat.numa_pages_migrated
13530 ± 6% +452.3% 74730 ± 4% proc-vmstat.numa_pte_updates
4444 ± 9% +476.1% 25606 ±101% proc-vmstat.pageoutrun
523627 ± 1% +67.6% 877482 ± 3% proc-vmstat.pgfault
1310 ± 14% +365.9% 6105 ± 11% proc-vmstat.pgmigrate_success
16384 ± 4% +72.5% 28257 ± 15% proc-vmstat.slabs_scanned
2452516 ± 62% +165.0% 6499322 ±123% latency_stats.avg.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_alloc_read_agfl.xfs_alloc_fix_freelist.xfs_alloc_vextent.xfs_ialloc_ag_alloc.xfs_dialloc.xfs_ialloc
619509 ±123% -100.0% 0.00 ± -1% latency_stats.avg.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_btree_read_buf_block.xfs_btree_lookup_get_block.xfs_btree_lookup.xfs_alloc_lookup_eq.xfs_alloc_fixup_trees.xfs_alloc_ag_vextent_size
668825 ±167% -74.7% 169025 ± 94% latency_stats.avg.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_btree_read_buf_block.xfs_btree_lookup_get_block.xfs_btree_lookup.xfs_inobt_insert.xfs_ialloc_ag_alloc.xfs_dialloc
0.00 ± -1% +Inf% 10533842 ±100% latency_stats.avg.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_da_read_buf.xfs_dir3_block_read.xfs_dir2_block_addname.xfs_dir_createname.xfs_create.xfs_generic_create
1434223 ± 33% +205.1% 4375501 ± 39% latency_stats.avg.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_imap_to_bp.xfs_iread.xfs_iget.xfs_ialloc.xfs_dir_ialloc.xfs_create
1134 ±173% +1.6e+05% 1818032 ±153% latency_stats.avg.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_read_agf.xfs_alloc_read_agf.xfs_alloc_fix_freelist.xfs_alloc_vextent.xfs_ialloc_ag_alloc.xfs_dialloc
35260 ± 6% +280.2% 134055 ± 12% latency_stats.hits.raid5_get_active_stripe.[raid456].raid5_make_request.[raid456].md_make_request.generic_make_request.submit_bio._xfs_buf_ioapply.xfs_buf_submit.xlog_bdstrat.xlog_sync.xlog_state_release_iclog._xfs_log_force_lsn.xfs_file_fsync
9385649 ± 3% +505.1% 56794939 ± 7% latency_stats.hits.raid5_get_active_stripe.[raid456].raid5_make_request.[raid456].md_make_request.generic_make_request.submit_bio.xfs_submit_ioend_bio.xfs_submit_ioend.xfs_vm_writepage.__writepage.write_cache_pages.generic_writepages.xfs_vm_writepages
3807550 ± 25% +155.1% 9711263 ± 73% latency_stats.max.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_alloc_read_agfl.xfs_alloc_fix_freelist.xfs_alloc_vextent.xfs_bmap_btalloc.xfs_bmap_alloc.xfs_bmapi_write
2688649 ± 59% +187.4% 7727816 ±117% latency_stats.max.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_alloc_read_agfl.xfs_alloc_fix_freelist.xfs_alloc_vextent.xfs_ialloc_ag_alloc.xfs_dialloc.xfs_ialloc
673991 ±108% -100.0% 0.00 ± -1% latency_stats.max.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_btree_read_buf_block.xfs_btree_lookup_get_block.xfs_btree_lookup.xfs_alloc_lookup_eq.xfs_alloc_fixup_trees.xfs_alloc_ag_vextent_size
668825 ±167% -74.6% 169683 ± 94% latency_stats.max.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_btree_read_buf_block.xfs_btree_lookup_get_block.xfs_btree_lookup.xfs_inobt_insert.xfs_ialloc_ag_alloc.xfs_dialloc
0.00 ± -1% +Inf% 14261726 ± 73% latency_stats.max.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_da_read_buf.xfs_dir3_block_read.xfs_dir2_block_addname.xfs_dir_createname.xfs_create.xfs_generic_create
1260860 ± 73% +328.3% 5400244 ±169% latency_stats.max.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_da_read_buf.xfs_dir3_leaf_read.xfs_dir2_leaf_addname.xfs_dir_createname.xfs_create.xfs_generic_create
5058489 ± 54% +148.5% 12570469 ± 76% latency_stats.max.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_da_read_buf.xfs_dir3_leaf_read.xfs_dir2_leaf_lookup_int.xfs_dir2_leaf_lookup.xfs_dir_lookup.xfs_lookup
1134 ±173% +4.8e+05% 5434961 ±154% latency_stats.max.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_read_agf.xfs_alloc_read_agf.xfs_alloc_fix_freelist.xfs_alloc_vextent.xfs_ialloc_ag_alloc.xfs_dialloc
6967127 ± 59% +82.8% 12733733 ±127% latency_stats.sum.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_alloc_read_agfl.xfs_alloc_fix_freelist.xfs_alloc_vextent.xfs_ialloc_ag_alloc.xfs_dialloc.xfs_ialloc
11217182 ± 27% +199.8% 33628397 ± 72% latency_stats.sum.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_alloc_read_agfl.xfs_alloc_fix_freelist.xfs_free_extent.xfs_trans_free_extent.xfs_bmap_finish.xfs_itruncate_extents
674368 ±108% -100.0% 0.00 ± -1% latency_stats.sum.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_btree_read_buf_block.xfs_btree_lookup_get_block.xfs_btree_lookup.xfs_alloc_lookup_eq.xfs_alloc_fixup_trees.xfs_alloc_ag_vextent_size
668825 ±167% -50.1% 333621 ± 97% latency_stats.sum.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_btree_read_buf_block.xfs_btree_lookup_get_block.xfs_btree_lookup.xfs_inobt_insert.xfs_ialloc_ag_alloc.xfs_dialloc
0.00 ± -1% +Inf% 51551059 ± 88% latency_stats.sum.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_da_read_buf.xfs_dir3_block_read.xfs_dir2_block_addname.xfs_dir_createname.xfs_create.xfs_generic_create
87014211 ± 98% +420.2% 4.527e+08 ± 36% latency_stats.sum.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_da_read_buf.xfs_dir3_block_read.xfs_dir2_block_lookup_int.xfs_dir2_block_lookup.xfs_dir_lookup.xfs_lookup
1318029 ± 69% +573.2% 8872791 ±169% latency_stats.sum.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_da_read_buf.xfs_dir3_leaf_read.xfs_dir2_leaf_addname.xfs_dir_createname.xfs_create.xfs_generic_create
60188444 ± 39% +368.4% 2.819e+08 ± 29% latency_stats.sum.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_imap_to_bp.xfs_iread.xfs_iget.xfs_ialloc.xfs_dir_ialloc.xfs_create
81840534 ± 50% +295.9% 3.24e+08 ± 29% latency_stats.sum.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_read_agf.xfs_alloc_read_agf.xfs_alloc_fix_freelist.xfs_alloc_vextent.xfs_bmap_btalloc.xfs_bmap_alloc
1134 ±173% +4.8e+05% 5435006 ±154% latency_stats.sum.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_read_agf.xfs_alloc_read_agf.xfs_alloc_fix_freelist.xfs_alloc_vextent.xfs_ialloc_ag_alloc.xfs_dialloc
1.69e+08 ± 42% +381.8% 8.144e+08 ± 23% latency_stats.sum.down.xfs_buf_lock._xfs_buf_find.xfs_buf_get_map.xfs_buf_read_map.xfs_trans_read_buf_map.xfs_read_agi.xfs_ialloc_read_agi.xfs_dialloc.xfs_ialloc.xfs_dir_ialloc.xfs_create
411.75 ±115% +3043.3% 12942 ± 21% latency_stats.sum.wait_on_page_bit.__migration_entry_wait.migration_entry_wait.do_swap_page.handle_mm_fault.__do_page_fault.do_page_fault.page_fault
70443 ± 38% +284.3% 270746 ± 59% latency_stats.sum.xlog_cil_force_lsn._xfs_log_force_lsn.xfs_file_fsync.vfs_fsync_range.do_fsync.SyS_fsync.entry_SYSCALL_64_fastpath
lkp-hsx02: Brickland Haswell-EX
Memory: 128G
uptime.boot
450 ++--------------------------------------------------------------------+
O O O O O O O O O O |
400 ++ O O O O O O O O O O O |
350 ++ |
| |
300 *+.*..*...*..*..*..*..* *..*..*..*..*...*..*..*..*..*..*...*..*..*
250 ++ : : |
| : : |
200 ++ : : |
150 ++ : : |
| : : |
100 ++ : : |
50 ++ : : |
| : |
0 ++-----------------------*--------------------------------------------+
uptime.idle
60000 ++-----------------------O------------------------------------------+
O O O O O O O O O O O O O |
50000 ++ O O O O O O O |
| |
| .*.. .*.. .*.. |
40000 *+.*..*..*. *...*..* *..*..*..*. *. *...*..*..*..*..*..*
| : : |
30000 ++ : : |
| : : |
20000 ++ : : |
| : : |
| : : |
10000 ++ : : |
| : |
0 ++-----------------------*------------------------------------------+
turbostat.Avg_MHz
300 ++--------------------------------------------------------------------+
O O O O |
250 ++ O |
| O O O O O O O O O O O O O O |
| O O |
200 ++ |
| |
150 ++ |
| |
100 ++ |
| |
*..*..*...*..*.. .*...*..*..*..*.. .*...*..*..*
50 ++ *..*..*.. .*..*..*..*. *. |
| .. |
0 ++-----------------------*--------------------------------------------+
turbostat._Busy
10 ++---------------------------------------------------------------------+
9 O+ O O O |
| O O O O O O |
8 ++ O O O O O O O O O O |
7 ++ O |
| |
6 ++ |
5 ++ |
4 ++ |
| |
3 ++ |
2 *+.*..*...*..*.. .*... *.. .*..*..*..*...*..*..*..*...*..*..*
| *. *.. + *...*. |
1 ++ + |
0 ++------------------------*--------------------------------------------+
turbostat.PkgWatt
400 ++--------------------------------------------------------------------+
O O O O O O O O O O O O O O O O O O O O O |
350 *+.*..*...*..*..*..*..* *..*..*..*..*...*..*..*..*..*..*...*..*..*
300 ++ : : |
| : : |
250 ++ : : |
| : : |
200 ++ : : |
| : : |
150 ++ : : |
100 ++ : : |
| : : |
50 ++ :: |
| : |
0 ++-----------------------*--------------------------------------------+
fsmark.app_overhead
1.4e+06 ++----------------------------O--------------------------O--------+
| |
1.2e+06 ++ O O O O |
O O O O O |
1e+06 ++ O O O O O O O O |
| O O |
800000 ++ |
| |
600000 *+.*..*..*..*..*..*..* *..*..*..*..*..*..*..*..*..*..*..*..*..*
| : : |
400000 ++ : : |
| : : |
200000 ++ : : |
| :: |
0 ++----------------------*-----------------------------------------+
fsmark.time.system_time
4500 ++-------------------------------------------------------------------+
| O |
4000 O+ O O |
3500 ++ O |
| O O O O O O O |
3000 ++ O O O O O O O |
2500 ++ O O |
| |
2000 ++ |
1500 ++ |
| |
1000 ++ |
500 ++ |
*..*..*..*...*..*..*..*.. .*..*...*..*..*..*..*..*..*..*...*..*..*..*
0 ++-----------------------*-------------------------------------------+
fsmark.time.percent_of_cpu_this_job_got
1400 ++-------------------------------------------------------------------+
| |
1200 O+ O O O |
| O O O O O O |
1000 ++ O O O O O O O O O |
| O O |
800 ++ |
| |
600 ++ |
| |
400 ++ |
| |
200 *+.*..*..*...*.. .*.. .*..*..*..*..*..*...*..*..*..*
| *. *.. .*..*...*..*. |
0 ++-----------------------*-------------------------------------------+
fsmark.time.elapsed_time
350 ++-----------------------O--------------------------------------------+
O O O O |
300 ++ O O O O O O O O O O O O O O O O |
| |
250 ++ |
| |
200 *+.*..*...*..*.. ..*.. .*.. .*...*..*..|
| *..*..* *..*..*..*..*. *. *..*. *
150 ++ : : |
| : : |
100 ++ : : |
| : : |
50 ++ : : |
| :: |
0 ++-----------------------*--------------------------------------------+
fsmark.time.elapsed_time.max
350 ++-----------------------O--------------------------------------------+
O O O O |
300 ++ O O O O O O O O O O O O O O O O |
| |
250 ++ |
| |
200 *+.*..*...*..*.. ..*.. .*.. .*...*..*..|
| *..*..* *..*..*..*..*. *. *..*. *
150 ++ : : |
| : : |
100 ++ : : |
| : : |
50 ++ : : |
| :: |
0 ++-----------------------*--------------------------------------------+
fsmark.time.minor_page_faults
100000 ++----------------------O-----O------------------------------------+
90000 ++ O |
O O O O O O O O |
80000 ++ O O O O O O |
70000 ++ O O O |
| O |
60000 ++ |
50000 ++ |
40000 ++ |
| |
30000 ++ |
20000 *+.*..*..*..*..*..*..* *..*...*..*..*..*..*..*..*..*..*..*..*..*
| + + |
10000 ++ + + |
0 ++----------------------*------------------------------------------+
fsmark.time.voluntary_context_switches
8e+07 ++------------------------------------------------------------------+
| O |
7e+07 O+ O O |
6e+07 ++ O O O O O |
| O O O O O O O O |
5e+07 ++ O O O O |
| |
4e+07 ++ |
| |
3e+07 ++ |
2e+07 ++ |
| |
1e+07 *+.*..*..*..*..*...*..*.. *..*..*..*..*..*..*..*...*..*..*..*..*..*
| .. |
0 ++-----------------------*------------------------------------------+
time.system_time
4500 ++-------------------------------------------------------------------+
| O |
4000 O+ O O |
3500 ++ O |
| O O O O O O O |
3000 ++ O O O O O O O |
2500 ++ O O |
| |
2000 ++ |
1500 ++ |
| |
1000 ++ |
500 ++ |
*..*..*..*...*..*..*..*.. .*..*...*..*..*..*..*..*..*..*...*..*..*..*
0 ++-----------------------*-------------------------------------------+
time.percent_of_cpu_this_job_got
1400 ++-------------------------------------------------------------------+
| |
1200 O+ O O O |
| O O O O O O |
1000 ++ O O O O O O O O O |
| O O |
800 ++ |
| |
600 ++ |
| |
400 ++ |
| |
200 *+.*..*..*...*.. .*.. .*..*..*..*..*..*...*..*..*..*
| *. *.. .*..*...*..*. |
0 ++-----------------------*-------------------------------------------+
time.elapsed_time
350 ++-----------------------O--------------------------------------------+
O O O O |
300 ++ O O O O O O O O O O O O O O O O |
| |
250 ++ |
| |
200 *+.*..*...*..*.. ..*.. .*.. .*...*..*..|
| *..*..* *..*..*..*..*. *. *..*. *
150 ++ : : |
| : : |
100 ++ : : |
| : : |
50 ++ : : |
| :: |
0 ++-----------------------*--------------------------------------------+
time.elapsed_time.max
350 ++-----------------------O--------------------------------------------+
O O O O |
300 ++ O O O O O O O O O O O O O O O O |
| |
250 ++ |
| |
200 *+.*..*...*..*.. ..*.. .*.. .*...*..*..|
| *..*..* *..*..*..*..*. *. *..*. *
150 ++ : : |
| : : |
100 ++ : : |
| : : |
50 ++ : : |
| :: |
0 ++-----------------------*--------------------------------------------+
time.minor_page_faults
100000 ++----------------------O-----O------------------------------------+
90000 ++ O |
O O O O O O O O |
80000 ++ O O O O O O |
70000 ++ O O O |
| O |
60000 ++ |
50000 ++ |
40000 ++ |
| |
30000 ++ |
20000 *+.*..*..*..*..*..*..* *..*...*..*..*..*..*..*..*..*..*..*..*..*
| + + |
10000 ++ + + |
0 ++----------------------*------------------------------------------+
time.voluntary_context_switches
8e+07 ++------------------------------------------------------------------+
| O |
7e+07 O+ O O |
6e+07 ++ O O O O O |
| O O O O O O O O |
5e+07 ++ O O O O |
| |
4e+07 ++ |
| |
3e+07 ++ |
2e+07 ++ |
| |
1e+07 *+.*..*..*..*..*...*..*.. *..*..*..*..*..*..*..*...*..*..*..*..*..*
| .. |
0 ++-----------------------*------------------------------------------+
[*] bisect-good sample
[O] bisect-bad sample
To reproduce:
git clone git://git.kernel.org/pub/scm/linux/kernel/git/wfg/lkp-tests.git
cd lkp-tests
bin/lkp install job.yaml # job file is attached in this email
bin/lkp run job.yaml
Disclaimer:
Results have been estimated based on internal Intel analysis and are provided
for informational purposes only. Any difference in system hardware or software
design or configuration may affect actual performance.
Thanks,
Xiaolong Ye
---
LKP_SERVER: inn
LKP_CGI_PORT: 80
LKP_CIFS_PORT: 139
testcase: fsmark
default-monitors:
wait: activate-monitor
kmsg:
uptime:
iostat:
heartbeat:
vmstat:
numa-numastat:
numa-vmstat:
numa-meminfo:
proc-vmstat:
proc-stat:
interval: 10
meminfo:
slabinfo:
interrupts:
lock_stat:
latency_stats:
softirqs:
bdi_dev_mapping:
diskstats:
nfsstat:
cpuidle:
cpufreq-stats:
turbostat:
pmeter:
sched_debug:
interval: 60
cpufreq_governor: performance
default-watchdogs:
oom-killer:
watchdog:
commit: 6ab2a4b806ae21b6c3e47c5ff1285ec06d505325
model: Brickland Haswell-EX
nr_cpu: 144
memory: 128G
hdd_partitions:
swap_partitions:
rootfs_partition:
category: benchmark
iterations: 1x
nr_threads: 64t
disk: 8BRD_12G
need_memory: 100G
md: RAID5
fs: xfs
fsmark:
filesize: 4M
test_size: 60G
sync_method: fsyncBeforeClose
queue: bisect
testbox: lkp-hsx02
tbox_group: lkp-hsx02
kconfig: x86_64-rhel
enqueue_time: 2016-03-10 14:41:55.378197851 +08:00
compiler: gcc-4.9
rootfs: debian-x86_64-2015-02-07.cgz
id: 29471207573211d88b6247f3d1cb5215dba98a6c
user: lkp
head_commit: 7811b4ffc31285e08ee7a6a96a94848c37a43e09
base_commit: f6cede5b49e822ebc41a099fe41ab4989f64e2cb
branch: linux-next/master
result_root: "/result/fsmark/performance-1x-64t-8BRD_12G-RAID5-xfs-4M-60G-fsyncBeforeClose/lkp-hsx02/debian-x86_64-2015-02-07.cgz/x86_64-rhel/gcc-4.9/6ab2a4b806ae21b6c3e47c5ff1285ec06d505325/0"
job_file: "/lkp/scheduled/lkp-hsx02/bisect_fsmark-performance-1x-64t-8BRD_12G-RAID5-xfs-4M-60G-fsyncBeforeClose-debian-x86_64-2015-02-07.cgz-x86_64-rhel-6ab2a4b806ae21b6c3e47c5ff1285ec06d505325-20160310-20183-1dvjl01-0.yaml"
max_uptime: 1208.5000000000002
initrd: "/osimage/debian/debian-x86_64-2015-02-07.cgz"
bootloader_append:
- root=/dev/ram0
- user=lkp
- job=/lkp/scheduled/lkp-hsx02/bisect_fsmark-performance-1x-64t-8BRD_12G-RAID5-xfs-4M-60G-fsyncBeforeClose-debian-x86_64-2015-02-07.cgz-x86_64-rhel-6ab2a4b806ae21b6c3e47c5ff1285ec06d505325-20160310-20183-1dvjl01-0.yaml
- ARCH=x86_64
- kconfig=x86_64-rhel
- branch=linux-next/master
- commit=6ab2a4b806ae21b6c3e47c5ff1285ec06d505325
- BOOT_IMAGE=/pkg/linux/x86_64-rhel/gcc-4.9/6ab2a4b806ae21b6c3e47c5ff1285ec06d505325/vmlinuz-4.5.0-rc5-00209-g6ab2a4b
- max_uptime=1208
- RESULT_ROOT=/result/fsmark/performance-1x-64t-8BRD_12G-RAID5-xfs-4M-60G-fsyncBeforeClose/lkp-hsx02/debian-x86_64-2015-02-07.cgz/x86_64-rhel/gcc-4.9/6ab2a4b806ae21b6c3e47c5ff1285ec06d505325/0
- LKP_SERVER=inn
- |-
hpet=disable noapic cpuidle.off=1
earlyprintk=ttyS0,115200 systemd.log_level=err
debug apic=debug sysrq_always_enabled rcupdate.rcu_cpu_stall_timeout=100
panic=-1 softlockup_panic=1 nmi_watchdog=panic oops=panic load_ramdisk=2 prompt_ramdisk=0
console=ttyS0,115200 console=tty0 vga=normal
rw
lkp_initrd: "/lkp/lkp/lkp-x86_64.cgz"
modules_initrd: "/pkg/linux/x86_64-rhel/gcc-4.9/6ab2a4b806ae21b6c3e47c5ff1285ec06d505325/modules.cgz"
bm_initrd: "/osimage/deps/debian-x86_64-2015-02-07.cgz/lkp.cgz,/osimage/deps/debian-x86_64-2015-02-07.cgz/run-ipconfig.cgz,/osimage/deps/debian-x86_64-2015-02-07.cgz/turbostat.cgz,/lkp/benchmarks/turbostat.cgz,/osimage/deps/debian-x86_64-2015-02-07.cgz/fs.cgz,/lkp/benchmarks/fsmark.cgz"
linux_headers_initrd: "/pkg/linux/x86_64-rhel/gcc-4.9/6ab2a4b806ae21b6c3e47c5ff1285ec06d505325/linux-headers.cgz"
repeat_to: 2
kernel: "/pkg/linux/x86_64-rhel/gcc-4.9/6ab2a4b806ae21b6c3e47c5ff1285ec06d505325/vmlinuz-4.5.0-rc5-00209-g6ab2a4b"
dequeue_time: 2016-03-10 15:02:00.906551604 +08:00
job_state: finished
loadavg: 36.99 30.11 13.65 2/1384 11794
start_time: '1457593437'
end_time: '1457593727'
version: "/lkp/lkp/.src-20160310-084410"
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu1/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu10/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu100/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu101/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu102/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu103/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu104/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu105/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu106/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu107/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu108/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu109/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu11/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu110/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu111/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu112/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu113/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu114/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu115/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu116/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu117/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu118/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu119/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu12/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu120/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu121/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu122/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu123/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu124/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu125/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu126/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu127/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu128/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu129/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu13/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu130/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu131/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu132/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu133/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu134/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu135/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu136/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu137/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu138/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu139/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu14/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu140/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu141/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu142/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu143/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu15/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu16/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu17/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu18/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu19/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu2/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu20/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu21/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu22/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu23/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu24/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu25/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu26/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu27/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu28/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu29/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu3/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu30/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu31/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu32/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu33/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu34/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu35/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu36/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu37/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu38/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu39/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu4/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu40/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu41/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu42/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu43/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu44/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu45/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu46/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu47/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu48/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu49/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu5/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu50/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu51/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu52/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu53/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu54/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu55/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu56/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu57/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu58/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu59/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu6/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu60/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu61/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu62/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu63/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu64/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu65/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu66/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu67/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu68/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu69/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu7/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu70/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu71/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu72/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu73/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu74/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu75/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu76/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu77/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu78/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu79/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu8/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu80/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu81/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu82/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu83/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu84/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu85/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu86/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu87/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu88/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu89/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu9/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu90/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu91/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu92/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu93/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu94/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu95/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu96/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu97/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu98/cpufreq/scaling_governor
2016-03-10 15:03:55 echo performance > /sys/devices/system/cpu/cpu99/cpufreq/scaling_governor
2016-03-10 15:03:56 mdadm -q --create /dev/md0 --chunk=256 --level=raid5 --raid-devices=8 --force --assume-clean /dev/ram0 /dev/ram1 /dev/ram2 /dev/ram3 /dev/ram4 /dev/ram5 /dev/ram6 /dev/ram7
2016-03-10 15:03:56 mkfs -t xfs /dev/md0
2016-03-10 15:03:56 mount -t xfs -o nobarrier,inode64 /dev/md0 /fs/md0
2016-03-10 15:03:58 ./fs_mark -d /fs/md0/1 -d /fs/md0/2 -d /fs/md0/3 -d /fs/md0/4 -d /fs/md0/5 -d /fs/md0/6 -d /fs/md0/7 -d /fs/md0/8 -d /fs/md0/9 -d /fs/md0/10 -d /fs/md0/11 -d /fs/md0/12 -d /fs/md0/13 -d /fs/md0/14 -d /fs/md0/15 -d /fs/md0/16 -d /fs/md0/17 -d /fs/md0/18 -d /fs/md0/19 -d /fs/md0/20 -d /fs/md0/21 -d /fs/md0/22 -d /fs/md0/23 -d /fs/md0/24 -d /fs/md0/25 -d /fs/md0/26 -d /fs/md0/27 -d /fs/md0/28 -d /fs/md0/29 -d /fs/md0/30 -d /fs/md0/31 -d /fs/md0/32 -d /fs/md0/33 -d /fs/md0/34 -d /fs/md0/35 -d /fs/md0/36 -d /fs/md0/37 -d /fs/md0/38 -d /fs/md0/39 -d /fs/md0/40 -d /fs/md0/41 -d /fs/md0/42 -d /fs/md0/43 -d /fs/md0/44 -d /fs/md0/45 -d /fs/md0/46 -d /fs/md0/47 -d /fs/md0/48 -d /fs/md0/49 -d /fs/md0/50 -d /fs/md0/51 -d /fs/md0/52 -d /fs/md0/53 -d /fs/md0/54 -d /fs/md0/55 -d /fs/md0/56 -d /fs/md0/57 -d /fs/md0/58 -d /fs/md0/59 -d /fs/md0/60 -d /fs/md0/61 -d /fs/md0/62 -d /fs/md0/63 -d /fs/md0/64 -n 240 -L 1 -S 1 -s 4194304