From: Franco B. <fr...@fu...> - 2005-10-12 02:56:56
|
An NFS over FUSE filesystem failure, but this time the machine is still up and the FUSE filesystem is still working normally. All the NFS daemons are in disk wait (D) except from one which appears to be sleeping (S) on request_wait_answer. Just before NFS failed the FUSE filesystem process was spinning at 100% CPU. I'm pretty sure the mremaps in the strace below are from the realloc in fill_dir_common which we've looked at before. The whole thing was just repeating continuously until NFS finally gave out and it went back to normal. The directory metadata/3d/010/AHL94_E96/pstm_input below contains about 75,000 files. mremap(0xb29f9000, 3145728, 3145728, MREMAP_MAYMOVE) = 0xb29f9000 mremap(0xb29f9000, 3145728, 3145728, MREMAP_MAYMOVE) = 0xb29f9000 mremap(0xb29f9000, 3145728, 3145728, MREMAP_MAYMOVE) = 0xb29f9000 mremap(0xb29f9000, 3145728, 3145728, MREMAP_MAYMOVE) = 0xb29f9000 mremap(0xb29f9000, 3145728, 3145728, MREMAP_MAYMOVE) = 0xb29f9000 mremap(0xb29f9000, 3145728, 3145728, MREMAP_MAYMOVE) = 0xb29f9000 mremap(0xb29f9000, 3145728, 3145728, MREMAP_MAYMOVE) = 0xb29f9000 mremap(0xb29f9000, 3145728, 3145728, MREMAP_MAYMOVE) = 0xb29f9000 getdents64(4, /* 0 entries */, 32768) = 0 mmap2(0xb347a000, 32768, PROT_NONE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb347a000 close(4) = 0 writev(3, [{"\20\20\0\0\0\0\0\0/\361\24\0\0\0\0\0", 16}, {"\377\377\377 \377\377\377\377\377\260m\27\0\0\0\0\0\f\0"..., 4096}], 2) = 4112 read(3, "(\0\0\0\3\0\0\0003\361\24\0\0\0\0\0\305\374\4\0\0\0\0\0"..., 8192) = 40 lstat64("/metadata/3d/010/AHL94_E96/pstm_input", {st_mode=S_IFDIR|0755, st_size=1740800, ...}) = 0 lstat64("/metadata/3d/010/AHL94_E96/pstm_input", {st_mode=S_IFDIR|0755, st_size=1740800, ...}) = 0 writev(3, [{"p\0\0\0\0\0\0\0003\361\24\0\0\0\0\0", 16}, {"\1\0\0\0\0\0\0 \0\0\0\0\0\0\0\0\0\305\374\4\0\0\0\0\0\0"..., 96}], 2) = 112 read(3, "(\0\0\0\3\0\0\0=\361\24\0\0\0\0\0\305\374\4\0\0\0\0\0\365"..., 8192) = 40 lstat64("/metadata/3d/010/AHL94_E96/pstm_input", {st_mode=S_IFDIR|0755, st_size=1740800, ...}) = 0 lstat64("/metadata/3d/010/AHL94_E96/pstm_input", {st_mode=S_IFDIR|0755, st_size=1740800, ...}) = 0 writev(3, [{"p\0\0\0\0\0\0\0=\361\24\0\0\0\0\0", 16}, {"\1\0\0\0\0\0\0\0 \0\0\0\0\0\0\0\0\305\374\4\0\0\0\0\0\0"..., 96}], 2) = 112 read(3, "0\0\0\0\33\0\0\0I\361\24\0\0\0\0\0\305\374\4\0\0\0\0\0"..., 8192) = 48 writev(3, [{" \0\0\0\0\0\0\0I\361\24\0\0\0\0\0", 16}, {"P\224E\263\0\0\0 \0\0\0\0\0\0\0\0\0", 16}], 2) = 32 read(3, "(\0\0\0\3\0\0\0Q\361\24\0\0\0\0\0\305\374\4\0\0\0\0\0\365"..., 8192) = 40 lstat64("/metadata/3d/010/AHL94_E96/pstm_input", {st_mode=S_IFDIR|0755, st_size=1740800, ...}) = 0 lstat64("/metadata/3d/010/AHL94_E96/pstm_input", {st_mode=S_IFDIR|0755, st_size=1740800, ...}) = 0 writev(3, [{"p\0\0\0\0\0\0\0Q\361\24\0\0\0\0\0", 16}, {"\1\0\0\0\0\0\0\0 \0\0\0\0\0\0\0\0\305\374\4\0\0\0\0\0\0"..., 96}], 2) = 112 read(3, "@\0\0\0\34\0\0\0^\361\24\0\0\0\0\0\305\374\4\0\0\0\0\0"..., 8192) = 64 writev(3, [{"\20\20\0\0\0\0\0\0^\361\24\0\0\0\0\0", 16}, {"\377\377\377 \377\377\377\377\377\30\217\27\0\0\0\0\0\f"..., 4096}], 2) = 4112 futex(0x80d1408, FUTEX_WAKE, 1) = 1 read(3, "(\0\0\0\3\0\0\0i\361\24\0\0\0\0\0\305\374\4\0\0\0\0\0\365"..., 8192) = 40 lstat64("/metadata/3d/010/AHL94_E96/pstm_input", {st_mode=S_IFDIR|0755, st_size=1740800, ...}) = 0 lstat64("/metadata/3d/010/AHL94_E96/pstm_input", {st_mode=S_IFDIR|0755, st_size=1740800, ...}) = 0 writev(3, [{"p\0\0\0\0\0\0\0i\361\24\0\0\0\0\0", 16}, {"\1\0\0\0\0\0\0\0 \0\0\0\0\0\0\0\0\305\374\4\0\0\0\0\0\0"..., 96}], 2) = 112 read(3, "0\0\0\0\33\0\0\0q\361\24\0\0\0\0\0\305\374\4\0\0\0\0\0"..., 8192) = 48 writev(3, [{" \0\0\0\0\0\0\0q\361\24\0\0\0\0\0", 16}, {"P\224E\263\0\0\0 \0\0\0\0\0\0\0\0\0", 16}], 2) = 32 read(3, "@\0\0\0\34\0\0\0z\361\24\0\0\0\0\0\305\374\4\0\0\0\0\0"..., 8192) = 64 open("/metadata/3d/010/AHL94_E96/pstm_input", O_RDONLY|O_NONBLOCK| O_LARGEFILE|O_DIRECTORY) = 4 fstat64(4, {st_mode=S_IFDIR|0755, st_size=1740800, ...}) = 0 fcntl64(4, F_SETFD, FD_CLOEXEC) = 0 getdents64(4, /* 1024 entries */, 32768) = 32752 getdents64(4, /* 1024 entries */, 32768) = 32768 getdents64(4, /* 1014 entries */, 32768) = 32768 mprotect(0xb347a000, 102400, PROT_READ|PROT_WRITE) = 0 getdents64(4, /* 990 entries */, 32768) = 32752 getdents64(4, /* 993 entries */, 32768) = 32768 mmap2(NULL, 204800, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb2dce000 mmap2(0xb3482000, 69632, PROT_NONE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb3482000 mremap(0xb2dce000, 204800, 204800, MREMAP_MAYMOVE) = 0xb2dce000 mremap(0xb2dce000, 204800, 204800, MREMAP_MAYMOVE) = 0xb2dce000 mremap(0xb2dce000, 204800, 204800, MREMAP_MAYMOVE) = 0xb2dce000 mremap(0xb2dce000, 204800, 204800, MREMAP_MAYMOVE) = 0xb2dce000 |