[SSI-devel] [ ssic-linux-Bugs-686748 ] Filesystem stacking deadlock
Brought to you by:
brucewalker,
rogertsang
From: SourceForge.net <no...@so...> - 2008-10-06 08:36:09
|
Bugs item #686748, was opened at 2003-02-14 15:10 Message generated for change (Settings changed) made by rogertsang You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=405834&aid=686748&group_id=32541 Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: Filesystem Group: v1.2.0 Status: Open Resolution: Accepted >Priority: 5 Private: No Submitted By: David Zafman (dzafman) Assigned to: Roger Tsang (rogertsang) Summary: Filesystem stacking deadlock Initial Comment: This has only been seen once. There are other processes wedged processes, which based on their stacks, are also blocked on the same inodes i_sem. This stack trace below shows that if the same processes recursively calls generic_file_write() twice, it will, of course, deadlock, leaving the i_sem locked on the inode. We might look into the Non-stop Clusters code, to see how we dealt with this issue before, although it might be somewhat OS specific. Stack traceback for pid 8 EBP EIP Function (args) 0xc11f7aa0 0xc011a9c8 schedule+0x478 (0x1, 0xc11f6000, 0xc30cfbe8, 0xc1ff5c08, 0xc3cd94ec) kernel .text 0xc0100000 0xc011a550 0xc011aa80 0xc11f7ac0 0xc0106199 __down+0x69 (0xc3cd94ec, 0xc11f7b78, 0xc0c5c3a0) kernel .text 0xc0100000 0xc0106130 0xc0106200 0xc11f7ad4 0xc010634b __down_failed+0xb (0xc11f7b58, 0xffffffff, 0xc3f96000, 0xc11f7b04, 0xc4859a04) kernel .text 0xc0100000 0xc0106340 0xc0106354 0xc0135868 _text_lock_filemap+0x27b kernel .text 0xc0100000 0xc01355ed 0xc0135880 0xc11f7ae8 0xc0135561 generic_file_write+0x21 (0xc11f7b58, 0xc3f96000, 0x1000, 0xc11f7b78, 0xc11f6000) kernel .text 0xc0100000 0xc0135540 0xc0135590 0xc11f7b04 0xc4859a04 [ext3]ext3_file_write+0x44 (0xc11f7b58, 0xc3f96000, 0x1000, 0xc11f7b78, 0xc11f7b58) ext3 .text 0xc4858060 0xc48599c0 0xc4859a10 0xc11f7bd4 0xc01e3859 cfsd_write+0xb9 (0xc2345360, 0x8a000, 0x0, 0xc3f96000, 0x1000) kernel .text 0xc0100000 0xc01e37a0 0xc01e38d0 0xc11f7c40 0xc01e012c cfs_proc_write+0x8c (0xc0e8bba0, 0x8a000, 0x0, 0x1000, 0xc3f96000) kernel .text 0xc0100000 0xc01e00a0 0xc01e0220 0xc11f7c70 0xc01ddf97 cfs_writepage_sync+0x57 (0x0, 0xc0e8bba0, 0xc111e238, 0x0, 0x1000) kernel .text 0xc0100000 0xc01ddf40 0xc01de020 0xc11f7c9c 0xc01de0be cfs_writepage+0x9e (0xc111e238, 0xc3c1bd90, 0xc11f6000, 0x3b, 0x256) kernel .text 0xc0100000 0xc01de020 0xc01de100 [1]more> 0xc11f7ccc 0xc0139c96 shrink_cache+0x1f6 (0x6, 0x3, 0x20, 0xc03c4908, 0x6) kernel .text 0xc0100000 0xc0139aa0 0xc0139e40 0xc11f7cf0 0xc0139f83 shrink_caches+0x53 (0x20, 0xc11f6000, 0x1d2, 0xc03c4908) kernel .text 0xc0100000 0xc0139f30 0xc0139fb0 0xc11f7d08 0xc0139fe3 try_to_free_pages+0x33 (0x0, 0x0, 0x0, 0xc03c4a84, 0x81) kernel .text 0xc0100000 0xc0139fb0 0xc013a010 0xc11f7d28 0xc013aa98 balance_classzone+0x68 (0xc11f7d40, 0xc03c4908, 0xc03c4a78, 0x1d2, 0xab) kernel .text 0xc0100000 0xc013aa30 0xc013ac00 0xc11f7d50 0xc013ad0a __alloc_pages+0x10a (0xc11f7da4) kernel .text 0xc0100000 0xc013ac00 0xc013ad70 0xc11f7d58 0xc013aa26 _alloc_pages+0x16 (0xc11f7d88, 0x0, 0x1000, 0xfffffff4, 0x0) kernel .text 0xc0100000 0xc013aa10 0xc013aa30 0xc0135259 __generic_file_write+0x419 (0xc11f7e38, 0xc2436000, 0x1000, 0xc11f7e58, 0xc11f7e38) kernel .text 0xc0100000 0xc0134e40 0xc0135540 0xc11f7dc8 0xc0135570 generic_file_write+0x30 (0xc11f7e38, 0xc2436000, 0x1000, 0xc11f7e58, 0xc11f6000) kernel .text 0xc0100000 0xc0135540 0xc0135590 0xc11f7de4 0xc4859a04 [ext3]ext3_file_write+0x44 (0xc11f7e38, 0xc2436000, 0x1000, 0xc11f7e58, 0xc11f7e38) ext3 .text 0xc4858060 0xc48599c0 0xc4859a10 0xc11f7eb4 0xc01e3859 cfsd_write+0xb9 (0x202, 0x202, 0xc11dd780, 0x0, 0x0) kernel .text 0xc0100000 0xc01e37a0 0xc01e38d0 0xc11f7e3c 0xc48090c8 [scsi_mod]scsi_queue_next_request+0x48 (0xc044aee0) scsi_mod .text 0xc4802060 0xc4809080 0xc48091a0 0xc11f7ef4 0xc0123aac bh_action+0x4c (0x8, 0x0, 0x1, 0xc041d500, 0xfffffffe) [1]more> kernel .text 0xc0100000 0xc0123a60 0xc0123af0 0xc11f7f10 0xc0123951 tasklet_hi_action+0x61 (0xc0e8bba0, 0xab000, 0x0, 0x1000, 0xc2436000) kernel .text 0xc0100000 0xc01238f0 0xc01239a0 0xc11f7f50 0xc01ddf97 cfs_writepage_sync+0x57 (0x0, 0xc0e8bba0, 0xc10a2f38, 0x0, 0x1000) kernel .text 0xc0100000 0xc01ddf40 0xc01de020 0xc11f7f7c 0xc01de0be cfs_writepage+0x9e (0xc10a2f38, 0xc0e8bc64, 0xc01de020, 0x0, 0x4) kernel .text 0xc0100000 0xc01de020 0xc01de100 0xc11f7fa0 0xc01324b8 filemap_fdatasync+0x88 (0xc0e8bc54, 0x0, 0xc326e464, 0xc326e400, 0xc11f6000) kernel .text 0xc0100000 0xc0132430 0xc0132510 0xc11f7fc4 0xc015623a sync_unlocked_inodes+0xaa (0xc11f6000) kernel .text 0xc0100000 0xc0156190 0xc01564d0 0xc11f7fd0 0xc0144938 sync_old_buffers+0x28 (0x30f00, 0xc11f6000, 0x30f00, 0xc115bf9c, 0xc0105000) kernel .text 0xc0100000 0xc0144910 0xc01449a0 0xc11f7fec 0xc0144cec kupdate+0x13c kernel .text 0xc0100000 0xc0144bb0 0xc0144d00 0xc0105ab6 kernel_thread+0x26 kernel .text 0xc0100000 0xc0105a90 0xc0105ad0 ---------------------------------------------------------------------- Comment By: Roger Tsang (rogertsang) Date: 2008-10-06 04:32 Message: Testing a bug fix. The fix is OS specific (for Linux kernel-2.6.11). This bug appears on 1.9.x (pre-1.9.6) by putting VM pressure on non-chard CFS stacked device mapper. The following was reproduced with CFS FLUSH_SYNC. Stack traceback for pid 252853 0xf3dd10c0 252853 248638 0 0 D 0xf3dd1280 rsync EBP EIP Function (args) 0xcf4b5318 0xc03d7bee schedule+0x31e (0xf3dd10c0, 0x1, 0xf3dd10c0, 0xc0119bd0, 0xd1483d30) 0xcf4b5350 0xc03d76c5 __down+0x75 (0x1, 0xc04d1880) 0xcf4b5360 0xc03d7866 __down_failed+0xa (0x0, 0xd1483cac, 0xcf4b5394, 0xc0280e80, 0xcf4b5444) 0xc01426a1 .text.lock.filemap+0x56 0xcf4b5394 0xc01424ca generic_file_writev+0x2a (0xcf4b53f8, 0xdf623aa0, 0x1, 0xcf4b54b0, 0xffffffff) 0xcf4b54a4 0xc028153a cfsd_write+0x1aa (0xcf02f540, 0x802, 0x0, 0xdf623aa0, 0x1) 0xcf4b5534 0xc027be80 cfs_proc_write+0x300 (0x7fe, 0xcf4b55d8, 0xe148ef00) 0xcf4b5580 0xc0142a39 mempool_alloc+0x69 (0xe148ef00, 0xcf4b5560, 0x8, 0x0, 0x0) 0xcf4b5580 0xc0277c88 cfs_async_handler_write+0x68 (0xcf4b55b0, 0xe148ef00, 0x7fe, 0x0, 0x0) 0xcf4b55c0 0xc02780a4 cfs_flush_one+0x184 (0xcf4b55d8, 0xe48bee6c, 0x0, 0xfffe0d59, 0xcf4b55d8) 0xcf4b55ec 0xc02781d1 cfs_flush_list+0x51 (0xcf4b5608, 0x8, 0x0, 0x0, 0xe48bed40) 0xcf4b561c 0xc0278838 cfs_flush_inode+0x88 (0xe48bee6c, 0x0, 0x0, 0x0, 0x0) 0xcf4b5650 0xc02771ac cfs_writepages+0xcc (0xe48bef1c, 0xcf4b575c, 0xc017ce1b) 0xcf4b5664 0xc01457a7 do_writepages+0x27 (0xe48bef1c, 0xcf4b575c, 0x0, 0x0, 0xf2fbb200) 0xcf4b568c 0xc0185990 __sync_single_inode+0x60 (0xe48bee6c, 0xcf4b575c, 0xd8a15aac, 0xcf9d853c, 0xcf9d853c) 0xcf4b56e8 0xc0185b95 __writeback_single_inode+0x35 (0xe48bee6c, 0xcf4b575c, 0xcf4b4000, 0xf2fbb29c, 0xf2fbb294) 0xcf4b5718 0xc0185e5a sync_sb_inodes+0x1aa (0xf2fbb200, 0xcf4b575c, 0x600, 0xcf4b575c, 0x2515b) 0xcf4b5734 0xc01861c6 writeback_inodes+0x106 (0xcf4b575c, 0xcf4b5754, 0xcf4b5758, 0xd1483d5c, 0xf61792b8) 0xcf4b57a8 0xc0145217 balance_dirty_pages+0x47 (0xd1483d5c, 0xd1483d5c, 0x7fe, 0xc146e6f0) [0]more> 0xcf4b57c0 0xc014538c balance_dirty_pages_ratelimited+0x6c (0xd1483d5c, 0xc146e6f0, 0x2, 0x800, 0x800) 0xcf4b5874 0xc0141679 generic_file_buffered_write+0x2e9 (0xcf4b593c, 0xe7abf7c0, 0x1, 0x1f2a7800, 0x0) 0xcf4b5924 0xc0141dbc __generic_file_aio_write_nolock+0x2cc (0xcf4b593c, 0xe7abf7c0, 0x1, 0xcf4b5b10, 0xf895633a) 0xcf4b59c0 0xc0142140 __generic_file_write_nolock+0x90 (0xcf4b5a58, 0xe7abf7c0, 0x1, 0xcf4b5b10, 0xcf4b5aa4) 0xcf4b59f4 0xc01424f6 generic_file_writev+0x56 (0xcf4b5a58, 0xe7abf7c0, 0x1, 0xcf4b5b10, 0xc0000000) 0xcf4b5b04 0xc028153a cfsd_write+0x1aa (0xcf02f540, 0x1f2a7002, 0x0, 0xe7abf7c0, 0x1) 0xcf4b5b94 0xc027be80 cfs_proc_write+0x300 (0x7fe, 0xcf4b5c38, 0xe148ef80) 0xcf4b5be0 0xc0142a39 mempool_alloc+0x69 (0xe148ef80, 0xcf4b5bc0, 0x8, 0x0, 0x0) 0xcf4b5be0 0xc0277c88 cfs_async_handler_write+0x68 (0xcf4b5c10, 0xe148ef80, 0x7fe, 0x0, 0x4) 0xcf4b5c20 0xc02780a4 cfs_flush_one+0x184 (0xcf4b5c38, 0xe48bee6c, 0x4, 0xffffffff, 0xcf4b5c38) 0xcf4b5c4c 0xc02781d1 cfs_flush_list+0x51 (0xcf4b5c68, 0x8, 0x4, 0x1, 0xe48bed40) 0xcf4b5c7c 0xc0278838 cfs_flush_inode+0x88 (0xe48bee6c, 0x1f2a7, 0x1, 0x4, 0x2) 0xcf4b5ca4 0xc0278959 cfs_sync_inode+0x59 (0xe48bee6c, 0x1f2a7, 0x1, 0x6, 0x3fd) 0xcf4b5cd0 0xc0277ba8 cfs_updatepage+0x158 (0xf46b9dc0, 0xc14acae0, 0x802, 0x3fd, 0xbff) 0xcf4b5d84 0xc0141603 generic_file_buffered_write+0x273 (0xcf4b5ee0, 0xcf4b5e58, 0x1, 0x1f2a7802, 0x0) 0xcf4b5e34 0xc0141dbc __generic_file_aio_write_nolock+0x2cc (0xcf4b5ee0, 0xcf4b5e58, 0x1, 0xcf4b5f1c, 0x0) 0xcf4b5e6c 0xc0142286 generic_file_aio_write+0x76 (0xcf4b5ee0, 0x80923fa, 0x3fd, 0x1f2a7802, 0x0) 0xcf4b5ea8 0xc0275756 __cfs_file_write+0xe6 (0xcf4b5ee0, 0x0, 0x80923fa, 0x3fd, 0xcf4b5ed8) 0xcf4b5ec4 0xc027580e cfs_file_aio_write+0x2e (0xcf4b5ee0, 0x80923fa, 0x3fd, 0x1f2a7802, 0x0) 0xcf4b5f6c 0xc015f7eb do_sync_write+0xab (0x0, 0x1, 0xffffffff, 0xf46b9dc0, 0xf4d7f754) ---------------------------------------------------------------------- Comment By: Roger Tsang (rogertsang) Date: 2005-09-29 01:19 Message: Logged In: YES user_id=1246761 Is this a loopback filesystem? ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=405834&aid=686748&group_id=32541 |