Re: [mod-security-users] httpd hangs after mod_security update
Brought to you by:
victorhora,
zimmerletw
Menu
▾
▴
Re: [mod-security-users] httpd hangs after mod_security update
|
From: Glen H. <gho...@in...> - 2011-03-22 23:48:18
|
Hi Matt,
Its the CPU getting out of control. I believe its the parent process
because no other processes spawn.
Im using the prefork mpm (See htttpd -l)
Heres a partial top.
last pid: 54896; load averages: 0.56, 0.15, 0.05
up 25+21:34:39 22:18:26
267 processes: 2 running, 265 sleeping
CPU: 49.3% user, 0.0% nice, 0.0% system, 0.0% interrupt, 50.7% idle
Mem: 172M Active, 508M Inact, 232M Wired, 6336K Cache, 111M Buf, 61M Free
Swap: 2012M Total, 2012M Free
PID USERNAME THR PRI NICE SIZE RES STATE C TIME WCPU
COMMAND
54895 root 1 114 0 46224K 10668K CPU1 1 0:47 88.96%
httpd
15397 root 221 44 0 1480M 112M select 0 230:38 0.00% java
970 nagios 1 44 0 9956K 2488K select 0 1:23 0.00%
nrpe2
1202 root 1 44 0 4772K 1404K kqread 1 0:18 0.00%
master
1204 postfix 1 44 0 4776K 1460K kqread 0 0:12 0.00% qmgr
1123 root 1 44 0 6920K 1348K nanslp 0 0:09 0.00% cron
746 root 1 44 0 5992K 1280K select 0 0:07 0.00%
syslogd
19443 ghollings 1 44 0 38064K 4676K select 0 0:03 0.00% sshd
19451 ghollings 1 44 0 37040K 3876K select 0 0:03 0.00% sshd
19455 root 1 44 0 9188K 2320K ttyin 1 0:01 0.00% bash
1088 mysql 6 44 0 63300K 11656K ucond 1 0:00 0.00%
mysqld
[root@dev /usr/local/src]# /usr/local/apache/bin/httpd -l
Compiled in modules:
core.c
mod_authn_file.c
mod_authn_default.c
mod_authz_host.c
mod_authz_groupfile.c
mod_authz_user.c
mod_authz_default.c
mod_auth_basic.c
mod_include.c
mod_filter.c
mod_deflate.c
mod_log_config.c
mod_env.c
mod_mime_magic.c
mod_expires.c
mod_headers.c
mod_usertrack.c
mod_unique_id.c
mod_setenvif.c
mod_version.c
mod_ssl.c
prefork.c
http_core.c
mod_mime.c
mod_status.c
mod_autoindex.c
mod_asis.c
mod_cgi.c
mod_negotiation.c
mod_dir.c
mod_actions.c
mod_speling.c
mod_userdir.c
mod_alias.c
mod_rewrite.c
mod_so.c
Heres a excerpted truss of the httpd process.. I hope this gives you the
info you are after. I still have no idea whats chewing cpu.
Please note that there were a stack of the 'libc' errors.
stat("/usr/share/nls/C/libc.cat",0x7fffffffe440) ERR#2 'No such file or
directory'
stat("/usr/share/nls/libc/C",0x7fffffffe440) ERR#2 'No such file or
directory'
stat("/usr/local/share/nls/C/libc.cat",0x7fffffffe440) ERR#2 'No such file
or directory'
stat("/usr/local/share/nls/libc/C",0x7fffffffe440) ERR#2 'No such file or
directory'
stat("/usr/share/nls/C/libc.cat",0x7fffffffe440) ERR#2 'No such file or
directory'
stat("/usr/share/nls/libc/C",0x7fffffffe440) ERR#2 'No such file or
directory'
stat("/usr/local/share/nls/C/libc.cat",0x7fffffffe440) ERR#2 'No such file
or directory'
stat("/usr/local/share/nls/libc/C",0x7fffffffe440) ERR#2 'No such file or
directory'
stat("/usr/share/nls/C/libc.cat",0x7fffffffe440) ERR#2 'No such file or
directory'
stat("/usr/share/nls/libc/C",0x7fffffffe440) ERR#2 'No such file or
directory'
stat("/usr/local/share/nls/C/libc.cat",0x7fffffffe440) ERR#2 'No such file
or directory'
stat("/usr/local/share/nls/libc/C",0x7fffffffe440) ERR#2 'No such file or
directory'
stat("/usr/share/nls/C/libc.cat",0x7fffffffe440) ERR#2 'No such file or
directory'
stat("/usr/share/nls/libc/C",0x7fffffffe440) ERR#2 'No such file or
directory'
stat("/usr/local/share/nls/C/libc.cat",0x7fffffffe440) ERR#2 'No such file
or directory'
stat("/usr/local/share/nls/libc/C",0x7fffffffe440) ERR#2 'No such file or
directory'
getpid(0xa,0x1e,0x1,0x74,0x74,0x803101538) = 1323 (0x52b)
open("/dev/crypto",O_RDWR,00) ERR#2 'No such file or
directory'
open("/dev/crypto",O_RDWR,00) ERR#2 'No such file or
directory'
mmap(0x0,1048576,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0x0) =
34412167168 (0x803200000)
socket(PF_INET6,SOCK_DGRAM,0) = 3 (0x3)
close(3) = 0 (0x0)
socket(PF_INET,SOCK_DGRAM,0) = 3 (0x3)
close(3) = 0 (0x0)
socket(PF_INET6,SOCK_STREAM,0) = 3 (0x3)
fcntl(3,F_GETFD,) = 0 (0x0)
fcntl(3,F_SETFD,FD_CLOEXEC) = 0 (0x0)
socket(PF_INET,SOCK_STREAM,0) = 4 (0x4)
fcntl(4,F_GETFD,) = 0 (0x0)
fcntl(4,F_SETFD,FD_CLOEXEC) = 0 (0x0)
stat("/etc/nsswitch.conf",{ mode=-rw-r--r--
,inode=49489,size=327,blksize=4096 }) = 0 (0x0)
open("/etc/nsswitch.conf",O_RDONLY,0666) = 5 (0x5)
ioctl(5,TIOCGETA,0xffffe2c0) ERR#25 'Inappropriate ioctl
for device'
fstat(5,{ mode=-rw-r--r-- ,inode=49489,size=327,blksize=4096 }) = 0 (0x0)
read(5,"#\n# nsswitch.conf(5) - name ser"...,4096) = 327 (0x147)
read(5,0x80321c000,4096) = 0 (0x0)
sigprocmask(SIG_BLOCK,SIGHUP|SIGINT|SIGQUIT|SIGKILL|SIGPIPE|SIGALRM|SIGTERM|
SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ
|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2,0x0) = 0 (0x0)
access("/usr/local/apache/lib/nss_compat.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/nss_compat.so.1",0) ERR#2 'No such file or
directory'
access("/lib/nss_compat.so.1",0) ERR#2 'No such file or
directory'
access("/usr/lib/nss_compat.so.1",0) ERR#2 'No such file or
directory'
access("/usr/lib/compat/nss_compat.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/nss_compat.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/compat/pkg/nss_compat.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/compat/nss_compat.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/mysql/nss_compat.so.1",0) ERR#2 'No such file or
directory'
access("/lib/nss_compat.so.1",0) ERR#2 'No such file or
directory'
access("/usr/lib/nss_compat.so.1",0) ERR#2 'No such file or
directory'
sigprocmask(SIG_SETMASK,0x0,0x0) = 0 (0x0)
sigprocmask(SIG_BLOCK,SIGHUP|SIGINT|SIGQUIT|SIGKILL|SIGPIPE|SIGALRM|SIGTERM|
SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ
|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2,0x0) = 0 (0x0)
access("/usr/local/apache/lib/nss_nis.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/nss_nis.so.1",0) ERR#2 'No such file or
directory'
access("/lib/nss_nis.so.1",0) ERR#2 'No such file or
directory'
access("/usr/lib/nss_nis.so.1",0) ERR#2 'No such file or
directory'
access("/usr/lib/compat/nss_nis.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/nss_nis.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/compat/pkg/nss_nis.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/compat/nss_nis.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/mysql/nss_nis.so.1",0) ERR#2 'No such file or
directory'
access("/lib/nss_nis.so.1",0) ERR#2 'No such file or
directory'
access("/usr/lib/nss_nis.so.1",0) ERR#2 'No such file or
directory'
sigprocmask(SIG_SETMASK,0x0,0x0) = 0 (0x0)
sigprocmask(SIG_BLOCK,SIGHUP|SIGINT|SIGQUIT|SIGKILL|SIGPIPE|SIGALRM|SIGTERM|
SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ
|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2,0x0) = 0 (0x0)
access("/usr/local/apache/lib/nss_files.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/nss_files.so.1",0) ERR#2 'No such file or
directory'
access("/lib/nss_files.so.1",0) ERR#2 'No such file or
directory'
access("/usr/lib/nss_files.so.1",0) ERR#2 'No such file or
directory'
access("/usr/lib/compat/nss_files.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/nss_files.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/compat/pkg/nss_files.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/compat/nss_files.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/mysql/nss_files.so.1",0) ERR#2 'No such file or
directory'
access("/lib/nss_files.so.1",0) ERR#2 'No such file or
directory'
access("/usr/lib/nss_files.so.1",0) ERR#2 'No such file or
directory'
sigprocmask(SIG_SETMASK,0x0,0x0) = 0 (0x0)
sigprocmask(SIG_BLOCK,SIGHUP|SIGINT|SIGQUIT|SIGKILL|SIGPIPE|SIGALRM|SIGTERM|
SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ
|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2,0x0) = 0 (0x0)
access("/usr/local/apache/lib/nss_dns.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/nss_dns.so.1",0) ERR#2 'No such file or
directory'
access("/lib/nss_dns.so.1",0) ERR#2 'No such file or
directory'
access("/usr/lib/nss_dns.so.1",0) ERR#2 'No such file or
directory'
access("/usr/lib/compat/nss_dns.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/nss_dns.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/compat/pkg/nss_dns.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/compat/nss_dns.so.1",0) ERR#2 'No such file or
directory'
access("/usr/local/lib/mysql/nss_dns.so.1",0) ERR#2 'No such file or
directory'
access("/lib/nss_dns.so.1",0) ERR#2 'No such file or
directory'
access("/usr/lib/nss_dns.so.1",0) ERR#2 'No such file or
directory'
sigprocmask(SIG_SETMASK,0x0,0x0) = 0 (0x0)
ioctl(5,TIOCGETA,0xffffe2d0) ERR#25 'Inappropriate ioctl
for device'
close(5) = 0 (0x0)
sigprocmask(SIG_BLOCK,SIGHUP|SIGINT|SIGQUIT|SIGKILL|SIGPIPE|SIGALRM|SIGTERM|
SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ
|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2,0x0) = 0 (0x0)
sigprocmask(SIG_SETMASK,0x0,0x0) = 0 (0x0)
sigprocmask(SIG_BLOCK,SIGHUP|SIGINT|SIGQUIT|SIGKILL|SIGPIPE|SIGALRM|SIGTERM|
SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ
|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2,0x0) = 0 (0x0)
sigprocmask(SIG_SETMASK,0x0,0x0) = 0 (0x0)
geteuid(0x8014d7eb0,0x30,0x0,0x7fffffffe618,0x801349d20,0x100) = 0 (0x0)
open("/etc/spwd.db",O_RDONLY,00) = 5 (0x5)
fcntl(5,F_SETFD,FD_CLOEXEC) = 0 (0x0)
fstat(5,{ mode=-rw------- ,inode=49853,size=40960,blksize=4096 }) = 0 (0x0)
read(5,"\0\^F\^Ua\0\0\0\^B\0\0\^D\M-R\0"...,260) = 260 (0x104)
pread(0x5,0x80321c000,0x1000,0x6000,0x1,0x0) = 4096 (0x1000)
pread(0x5,0x803235000,0x1000,0x4000,0x1,0x0) = 4096 (0x1000)
pread(0x5,0x803236000,0x1000,0x5000,0x1,0x0) = 4096 (0x1000)
pread(0x5,0x803237000,0x1000,0x7000,0x1,0x0) = 4096 (0x1000)
pread(0x5,0x803238000,0x1000,0x8000,0x1,0x0) = 4096 (0x1000)
pread(0x5,0x803239000,0x1000,0x1000,0x1,0x0) = 4096 (0x1000)
pread(0x5,0x80323a000,0x1000,0x2000,0x1,0x0) = 4096 (0x1000)
pread(0x5,0x80323b000,0x1000,0x3000,0x1,0x0) = 4096 (0x1000)
close(5) = 0 (0x0)
stat("/etc/nsswitch.conf",{ mode=-rw-r--r--
,inode=49489,size=327,blksize=4096 }) = 0 (0x0)
open("/etc/group",O_RDONLY,0666) = 5 (0x5)
fstat(5,{ mode=-rw-r--r-- ,inode=49351,size=620,blksize=4096 }) = 0 (0x0)
lseek(5,0x0,SEEK_CUR) = 0 (0x0)
lseek(5,0x0,SEEK_SET) = 0 (0x0)
read(5,"# $FreeBSD: src/etc/group,v 1.35"...,4096) = 620 (0x26c)
close(5) = 0 (0x0)
stat("/usr/local/apache/htdocs",{ mode=drwxr-xr-x
,inode=1601973,size=512,blksize=4096 }) = 0 (0x0)
open("/var/log/modsecurity/audit.log",O_WRONLY|O_APPEND|O_CREAT,0640) = 5
(0x5)
fcntl(5,F_GETFD,) = 0 (0x0)
fcntl(5,F_SETFD,FD_CLOEXEC) = 0 (0x0)
open("/var/log/modsecurity/debug.log",O_WRONLY|O_APPEND|O_CREAT,0640) = 6
(0x6)
fcntl(6,F_GETFD,) = 0 (0x0)
fcntl(6,F_SETFD,FD_CLOEXEC) = 0 (0x0)
open("/etc/asl/whitelist",O_RDONLY,00) = 7 (0x7)
read(7,0x803237028,4096) = 0 (0x0)
close(7) = 0 (0x0)
open("/etc/asl/whitelist",O_RDONLY,00) = 7 (0x7)
read(7,0x80323b340,4096) = 0 (0x0)
close(7) = 0 (0x0)
removing /etc/asl/whitelist only makes the config error out. Editing it
makes no difference other than seeing the entries in the truss.
Thanks for your response.
Glen
-----Original Message-----
From: matthew sporleder [mailto:msp...@gm...]
Sent: Tuesday, 22 March 2011 11:14 PM
To: gho...@in...
Cc: mod...@li...
Subject: Re: [mod-security-users] httpd hangs after mod_security update
On Tue, Mar 22, 2011 at 1:48 AM, Glen Hollings <gho...@in...>
wrote:
> After days of frustration, Im reaching out J
>
>
>
> Because of the addition of decodeBase64Ext, I obviously needed to
> update modsecurity. But once I updated from 2.5.11 to .13, httpd no
> longer completes startup, and eventually chews 100% of the CPU, and
> needs to be cancelled.
>
>
>
> I am running
>
>
>
> FreeBSD 8.0
>
> Httpd 2.2.17 (Have tried 2.2.15) (I have tried compiling this with
> external pcre with no luck)
>
> Php 5.2.3
>
>
>
> Through a process of trial and much error I am also running these
> (although they didnt change the behaviour at all)
>
>
>
> Pcre 8.12
>
> APR 1.4.2
>
> APR-Util 1.3.10
>
>
>
> Modsec 2.5.11 runs perfectly, even recompiling it in the updated
> environment it works fine.
>
>
>
> I tried modsec 2.5.12 and it has the same issues. I have also tried
> compiling modsec with the pcre that comes with httpd with no change.
>
>
>
> I have googled around a heap and found a number of similar issues, but
> unfortunately with no fix.
>
>
>
>
>
> Running httpd with debugging enabled doesnt give me anything useful
>
>
>
> [root@dev /usr/local/src/modsecurity-apache_2.5.13/apache2]#
> /usr/local/apache/bin/apachectl -e debug
>
> [Tue Mar 22 05:40:55 2011] [debug] mod_so.c(246): loaded module
> php5_module
>
> [Tue Mar 22 05:40:55 2011] [debug] mod_so.c(246): loaded module
> security2_module
>
>
>
>
>
> This is what lead me to change pcre, but hey, im not exactly sure how
> to use gdb
>
>
>
> [root@dev /usr/local/src]# gdb -p 52455 /usr/local/apache/bin/httpd
>
> GNU gdb 6.1.1 [FreeBSD]
>
> Copyright 2004 Free Software Foundation, Inc.
>
> GDB is free software, covered by the GNU General Public License, and
> you are
>
> welcome to change it and/or distribute copies of it under certain
> conditions.
>
> Type "show copying" to see the conditions.
>
> There is absolutely no warranty for GDB. Type "show warranty" for
details.
>
> This GDB was configured as "amd64-marcel-freebsd"...
>
> Attaching to program: /usr/local/apache/bin/httpd, process 52455
>
> Reading symbols from /lib/libz.so.5...done.
>
> Loaded symbols for /lib/libz.so.5
>
> Reading symbols from /usr/lib/libssl.so.6...done.
>
> Loaded symbols for /usr/lib/libssl.so.6
>
> Reading symbols from /lib/libcrypto.so.6...done.
>
> Loaded symbols for /lib/libcrypto.so.6
>
> Reading symbols from /lib/libm.so.5...done.
>
> Loaded symbols for /lib/libm.so.5
>
> Reading symbols from /usr/local/apache/lib/libaprutil-1.so.3...done.
>
> Loaded symbols for /usr/local/apache/lib/libaprutil-1.so.3
>
> Reading symbols from /usr/local/lib/libexpat.so.6...done.
>
> Loaded symbols for /usr/local/lib/libexpat.so.6
>
> Reading symbols from /usr/local/apache/lib/libapr-1.so.4...done.
>
> Loaded symbols for /usr/local/apache/lib/libapr-1.so.4
>
> Reading symbols from /lib/libcrypt.so.5...done.
>
> Loaded symbols for /lib/libcrypt.so.5
>
> Reading symbols from /lib/libthr.so.3...done.
>
> [New Thread 8015021c0 (LWP 100466)]
>
> Loaded symbols for /lib/libthr.so.3
>
> Reading symbols from /lib/libc.so.7...done.
>
> Loaded symbols for /lib/libc.so.7
>
> Reading symbols from /usr/local/apache/modules/libphp5.so...done.
>
> Loaded symbols for /usr/local/apache/modules/libphp5.so
>
> Reading symbols from /usr/local/lib/libmcrypt.so.8...done.
>
> Loaded symbols for /usr/local/lib/libmcrypt.so.8
>
> Reading symbols from /usr/local/lib/libltdl.so.7...done.
>
> Loaded symbols for /usr/local/lib/libltdl.so.7
>
> Reading symbols from /usr/local/lib/libintl.so.8...done.
>
> Loaded symbols for /usr/local/lib/libintl.so.8
>
> Reading symbols from /usr/local/lib/libpng.so.6...done.
>
> Loaded symbols for /usr/local/lib/libpng.so.6
>
> Reading symbols from /usr/local/lib/libjpeg.so.11...done.
>
> Loaded symbols for /usr/local/lib/libjpeg.so.11
>
> Reading symbols from /usr/local/lib/libcurl.so.6...done.
>
> Loaded symbols for /usr/local/lib/libcurl.so.6
>
> Reading symbols from /usr/local/lib/mysql/libmysqlclient.so.16...done.
>
> Loaded symbols for /usr/local/lib/mysql/libmysqlclient.so.16
>
> Reading symbols from /usr/local/lib/libxml2.so.5...done.
>
> Loaded symbols for /usr/local/lib/libxml2.so.5
>
> Reading symbols from /usr/local/lib/libiconv.so.3...done.
>
> Loaded symbols for /usr/local/lib/libiconv.so.3
>
> Reading symbols from /usr/local/apache/modules/mod_security2.so...done.
>
> Loaded symbols for /usr/local/apache/modules/mod_security2.so
>
> Reading symbols from /usr/local/lib/libpcre.so.0...done.
>
> Loaded symbols for /usr/local/lib/libpcre.so.0
>
> Reading symbols from /usr/local/lib/liblua-5.1.so.1...done.
>
> Loaded symbols for /usr/local/lib/liblua-5.1.so.1
>
> Reading symbols from /libexec/ld-elf.so.1...done.
>
> Loaded symbols for /libexec/ld-elf.so.1
>
> [Switching to Thread 8015021c0 (LWP 100466)]
>
> 0x0000000802c5a729 in find_minlength () from
> /usr/local/lib/libpcre.so.0
>
>
>
>
>
> It seems to me that something fundamental has changed in 2.5.12+ that
> is making it difficult for FreeBSD somehow
>
>
Are you getting a crash or is your cpu just spinning out of control?
Is it the apache parent, or one of the children? Which mpm are you using?
It might be better to ktrace/dtruss the offending pids to see what they're
doing to use up all your cycles.
Matt
|