#594 Random Crash

1.8.x
open
core (110)
5
2015-02-25
2012-11-28
No

We experienced a seemingly random crash on our server which has been running for months without a single issue.

The following kernel error was logged:
kernel: [8988656.469074] opensips[18871]: segfault at 0 ip 000000000048c0d9 sp 00007fffd71a92c0 error 6 in opensips[400000+11c000]

Here is the full backtrace:
#0 0x000000000048c0d9 in fm_remove_free (n=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:172
pf = 0x0
hash = 1
#1 fm_malloc (qm=0x7f412fdd2000, size=8) at mem/f_malloc.c:378
frag = 0x7f41302c3300
n = <optimized out>
hash = <optimized out>
__FUNCTION__ = "fm_malloc"
#2 0x00007f4134e9f41c in shm_malloc (size=<optimized out>) at ../../evi/../mem/shm_mem.h:202
p = <optimized out>
#3 dlg_add_leg_info (dlg=0x7f41302f49d0, tag=0x7f4138515c98, rr=<optimized out>, contact=<optimized out>, cseq=0x7fffd71a9450, sock=<optimized out>,
mangled_from=0x0, mangled_to=0x0) at dlg_hash.c:343
leg = 0x7f41300b4c68
head = 0x0
rrp = <optimized out>
__FUNCTION__ = "dlg_add_leg_info"
#4 0x00007f4134e9a8e6 in init_leg_info (dlg=0x7f41302f49d0, msg=0x7f4138504718, t=<optimized out>, tag=<optimized out>, mangled_from=<optimized out>,
mangled_to=0x0) at dlg_handlers.c:243
skip_recs = 0
cseq = {
s = 0x741f7f "2 INVITE\r\nExpires: 180\r\nFrom: <sip:3226@41.73.53.1>;tag=c0a8010a-ba80\r\nMax-Forwards: 69\r\nProxy-Authorization: Digest realm=\"41.73.53.1\", nonce=\"50b4902500005e9c4ceb3b7d9055f4453382581e2965bb0b\", usern"..., len = 1}
contact = {s = 0x7f4138507c70 "sip:3226@196.25.214.82:5060", len = 27}
rr_set = {s = 0x0, len = 0}
__FUNCTION__ = "init_leg_info"
#5 0x00007f4134e9b13e in dlg_create_dialog (t=0x0, req=0x7f4138504718, flags=0) at dlg_handlers.c:887
dlg = <optimized out>
s = {
s = 0x741ef3 "call-F113832A-B71A-3010-101F-4232@192.168.1.10\r\nContact: <sip:3226@192.168.1.10>\r\nContent-Length: 298\r\nContent-Type: application/sdp\r\nCSeq: 2 INVITE\r\nExpires: 180\r\nFrom: <sip:3226@41.73.53.1>;tag=c0a8"..., len = 46}
__FUNCTION__ = "dlg_create_dialog"
#6 0x00007f4134e88306 in w_create_dialog (req=0x7f4138504718) at dialog.c:832
t = <optimized out>
#7 0x0000000000413bf5 in do_action (a=0x7f41384908e8, msg=0x7f4138504718) at action.c:1483
val_s = {s = 0x4 <Address 0x4 out of bounds>, len = 7610407}
increment = <optimized out>
decrement = <optimized out>
aux = {s = 0x7f4138504718 "\r\214\002", len = 7610407}
adv_addr = {s = 0x7423b5 "", len = 4780225}
ret = <optimized out>
v = <optimized out>
to = <optimized out>
p = <optimized out>
tmp = <optimized out>
new_uri = <optimized out>
end = <optimized out>
crt = <optimized out>
len = <optimized out>
i = <optimized out>
user = <optimized out>
expires = <optimized out>
vals = {{
s = 0x74223e "39f1e0281836c06018033323236700581303131391301000f0b413032312d303133383633\r\n\r\nv=0\r\no=Quintum 5362 5362 IN IP4 192.168.1.10\r\ns=VoipCall\r\nc=IN IP4 192.168.1.10\r\nt=0 0\r\nm=audio 10412 RTP/AVP 18 8 101 13\r\n"..., len = 7610112}, {s = 0x7fffd71a9a00 "\265#t", len = -1}, {
s = 0x7f4138512c98 <incomplete sequence \350>, len = 944855384}, {
s = 0x742164 "branch=z9hG4bK-tenor-c0a8-010a-23383\r\nQuintum: 0c01030b053235353330050100075b1b01010000211b3600040a8090809", '0' <repeats 11 times>, "8300180c818080810000800380000000001e03808183006c08808180803332323600700680813031313"..., len = 7610247}, {s = 0x7423b5 "", len = 4797239}}
result = {s = 0x2 <Address 0x2 out of bounds>, len = 7610694}
uri = {user = {s = 0x0, len = 816207616}, passwd = {s = 0x7f4135f1fb48 "acc", len = 3}, host = {s = 0x38506f3e <Address 0x38506f3e out of bounds>,
len = 2}, port = {s = 0xe <Address 0xe out of bounds>, len = 944785176}, params = {s = 0x7f41301fe1a8 "\002", len = 128}, headers = {
s = 0x71d5a8 "\b\244\335/A\177", len = 816205752}, port_no = 30936, proto = 12429, type = 32577, transport = {
s = 0x7f4136fb198a "\351\274\375\377\377\220H\213\005Y\372!", len = -686122836}, ttl = {s = 0x3 <Address 0x3 out of bounds>, len = -686122836},
user_param = {s = 0x7f4138488c60 "H\201N8A\177", len = -686122912}, maddr = {s = 0x7fffd71a9898 "\254\004C", len = 814577880}, method = {
s = 0x751ee0 "\270\255\n0A\177", len = 816205752}, lr = {s = 0x7f4138504718 "\r\214\002", len = 200}, r2 = {s = 0x7f41301fe1b8 "",
len = -686122528}, gr = {s = 0x7f41308d79f0 "\001", len = 944278552}, transport_val = {s = 0x4304ac "H\205\300\017\205X\377\377\377H\213=LH3",
len = 944785176}, ttl_val = {s = 0x7f41308d78d8 "", len = 814584496}, user_param_val = {s = 0x400 <Address 0x400 out of bounds>, len = 1},
maddr_val = {s = 0x0, len = 944859768}, method_val = {s = 0x7f4136fb1a90 "L\213\253", <incomplete sequence \360>, len = 814578008}, lr_val = {
s = 0x33c <Address 0x33c out of bounds>, len = 944859768}, r2_val = {s = 0x7f41308d79f8 "\002", len = 1}, gr_val = {
s = 0x7fffd71a9918 "\215\326\372\066A\177", len = 944859768}, u_name = {{s = 0x7f4136fad68d "\203\370\377\211\303\017\204P\001", len = 806659848},
{s = 0x0, len = 814577880}, {s = 0x7f4138504718 "\r\214\002", len = 0}, {s = 0x7f4130882238 "", len = 46717}, {
s = 0x1 <Address 0x1 out of bounds>, len = 7610463}}, u_val = {{s = 0x7f413850caa0 "\001", len = 7610386}, {
s = 0x742164 "branch=z9hG4bK-tenor-c0a8-010a-23383\r\nQuintum: 0c01030b053235353330050100075b1b01010000211b3600040a8090809", '0' <repeats 11 times>, "8300180c818080810000800380000000001e03808183006c08808180803332323600700680813031313"..., len = 944818848}, {
s = 0x4ae3ad "H\205\300H\211D$PD\017\266T$8L\213\\$@\017\204\340\036", len = 29}, {
s = 0x49ff62 "H\205\300I\211\306L\213T$ D\213\\$(\017\204M\a", len = -686122244}, {s = 0x7f4138504f38 "\220\001", len = 0}}, u_params_no = 22872}
next_hop = {user = {s = 0x4c4b40 "\211\350A\207\004$\205\300t\036\205\333~\n\203\353\001\353\355\017\037D", len = 376}, passwd = {
s = 0x71d5a8 "\b\244\335/A\177", len = 944781648}, host = {
s = 0x7525e0 "insert into dialog (hash_entry,hash_id,callid,from_uri,from_tag,to_uri,to_tag,caller_sock,callee_sock,start_time,mangled_from_uri,mangled_to_uri,state,timeout,caller_cseq,callee_cseq,caller_ping_cseq,"..., len = 942047158}, port = {s = 0x0, len = 25}, params = {s = 0x7fffd71a96b0 "\017",
len = -686123344}, headers = {s = 0x7f41384e91b0 "\004", len = 816205752}, port_no = 7, proto = 0, type = ERROR_URI_T, transport = {
s = 0x7f4138504717 "", len = 0}, ttl = {s = 0x7f413850a0e8 "", len = 944781648}, user_param = {s = 0x0, len = 889960096}, maddr = {
s = 0x7f41385098e8 "\350\240P8A\177", len = 944806184}, method = {s = 0x7fffd71a9680 "\006\220\264P", len = 7}, lr = {
s = 0x7fffd71a9690 "\030GP8A\177", len = 816205752}, r2 = {s = 0x7f4135f19641 "\205\300t\232H\213\005dq ", len = 1354010630}, gr = {
s = 0x92a88 <Address 0x92a88 out of bounds>, len = 1354010630}, transport_val = {s = 0x8dd1c <Address 0x8dd1c out of bounds>, len = 944785176},
ttl_val = {s = 0x0, len = 0}, user_param_val = {s = 0x7f4138503950 "\360C\024\066A\177", len = 15}, maddr_val = {s = 0x7f4136142040 "\004",
len = 907287904}, method_val = {s = 0x7f4136144300 "", len = 907297472}, lr_val = {s = 0x7f4138269a37 "H\307C\b", len = 907288640}, r2_val = {
s = 0x7f4136141220 "\300\036t", len = 907288640}, gr_val = {s = 0xf <Address 0xf out of bounds>, len = 816205752}, u_name = {{
s = 0x7f4135f143b4 "\205\300\017\211\361\376\377\377H\213\005\355\303 ", len = 809199768}, {s = 0xe <Address 0xe out of bounds>,
len = 944785176}, {s = 0x7fffd71a97f0 "\250\341\037\060A\177", len = 7610111}, {s = 0x7f4130a64fb8 "\267\213\002", len = 907297728}, {
s = 0x7f41308d78d8 "", len = 944785176}}, u_val = {{s = 0x7f4136120be0 "H\373\361\065A\177", len = 200}, {
s = 0x7f4135f1b9e2 "\351\270\372\377\377f\017\037\204", len = 807924736}, {s = 0x7f41358fa610 "\017\234o5A\177", len = 39}, {
s = 0x7f4134e9d19a "H\215H\377H\211\302\061\300H\203\371\375w\323H\213\202\030\032", len = 0}, {
s = 0x7f41356f38fd "H\205\300H\211\305\017\204\257", len = 39}}, u_params_no = 0}
u = <optimized out>
port = <optimized out>
cmatch = <optimized out>
aitem = <optimized out>
adefault = <optimized out>
spec = <optimized out>
model = <optimized out>
val = {rs = {
s = 0x742029 "55f4453382581e2965bb0b\", username=\"federalmogul\", uri=\"sip:0119810000@41.73.53.1\", response=\"626090ba2fd3b2d919f697c330e26f37\"\r\nSession-GUID: 892363316-1633957986-875705136-4\r\nSupported: 100rel\r\nTo: <"..., len = 7610699}, ri = 944795136, flags = 32577}
pve = <optimized out>
name_s = {s = 0x7f4138504718 "\r\214\002", len = 7610694}
start = {tv_sec = 1354010631, tv_usec = 934753}
end_time = <optimized out>
__FUNCTION__ = "do_action"
#8 0x00000000004122ab in run_action_list (a=<optimized out>, msg=0x7f4138504718) at action.c:143
ret = 1
t = 0x7f41384908e8
__FUNCTION__ = "run_action_list"
#9 0x0000000000417a30 in run_actions (msg=<optimized out>, a=<optimized out>) at action.c:123
No locals.
#10 run_top_route (a=0x7f413848f098, msg=0x7f4138504718) at action.c:189
bl_last_msg_id = 166925
bk_action_flags = 0
bk_rec_lev = 0
#11 0x000000000044c5c7 in receive_msg (
buf=0x741ec0 "INVITE sip:0119810000@41.73.53.1 SIP/2.0\r\nCall-ID: call-F113832A-B71A-3010-101F-4232@192.168.1.10\r\nContact: <sip:3226@192.168.1.10>\r\nContent-Length: 298\r\nContent-Type: application/sdp\r\nCSeq: 2 INVITE\r"..., len=1269, rcv_info=0x7fffd71a9c90) at receive.c:165
msg = 0x7f4138504718
start = {tv_sec = 1354010631, tv_usec = 934360}
__FUNCTION__ = "receive_msg"
#12 0x0000000000487083 in udp_rcv_loop () at udp_server.c:424
len = 1269
buf = "INVITE sip:0119810000@41.73.53.1 SIP/2.0\r\nCall-ID: call-F113832A-B71A-3010-101F-4232@192.168.1.10\r\nContact: <sip:3226@192.168.1.10>\r\nContent-Length: 298\r\nContent-Type: application/sdp\r\nCSeq: 2 INVITE\r"...
tmp = <optimized out>
from = <optimized out>
fromlen = 16
ri = {src_ip = {af = 2, len = 4, u = {addrl = {1389763012, 140736802233736}, addr32 = {1389763012, 0, 3608845704, 32767}, addr16 = {6596, 21206, 0, 0,
40328, 55066, 32767, 0}, addr = "\304\031\326R\000\000\000\000\210\235\032\327\377\177\000"}}, dst_ip = {af = 2, len = 4, u = {addrl = {
20269353, 0}, addr32 = {20269353, 0, 0, 0}, addr16 = {18729, 309, 0, 0, 0, 0, 0, 0}, addr = ")I5\001", '\000' <repeats 11 times>}},
src_port = 5060, dst_port = 5060, proto = 1, proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s = {sa_family = 2,
sa_data = "\023\304\304\031\326R\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 50195, sin_addr = {s_addr = 1389763012},
sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 50195, sin6_flowinfo = 1389763012, sin6_addr = {__in6_u = {
__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}},
bind_address = 0x7f4138488c18}
p = 0x4f5
msg = {
s = 0x741ec0 "INVITE sip:0119810000@41.73.53.1 SIP/2.0\r\nCall-ID: call-F113832A-B71A-3010-101F-4232@192.168.1.10\r\nContact: <sip:3226@192.168.1.10>\r\nContent-Length: 298\r\nContent-Type: application/sdp\r\nCSeq: 2 INVITE\r"..., len = 1269}
__FUNCTION__ = "udp_rcv_loop"
#13 0x0000000000429a46 in main_loop () at main.c:876
i = 3
pid = <optimized out>
si = 0x7f4138488c18
startup_done = 0x0
chd_rank = 4
load_p = 0x7f41303d4370
#14 main (argc=<optimized out>, argv=<optimized out>) at main.c:1520
cfg_log_stderr = <optimized out>
cfg_stream = <optimized out>
c = <optimized out>
r = <optimized out>
tmp = 0x7fffd71aaf16 ""
tmp_len = <optimized out>
port = <optimized out>
proto = <optimized out>
ret = <optimized out>
seed = 2837169747
rfd = <optimized out>
__FUNCTION__ = "main"

Version info:

version: opensips 1.8.0-notls (x86_64/linux)
flags: STATS: Off, USE_IPV6, USE_TCP, DISABLE_NAGLE, USE_MCAST, SHM_MEM, SHM_MMAP, PKG_MALLOC, F_MALLOC, FAST_LOCK-ADAPTIVE_WAIT
ADAPTIVE_WAIT_LOOPS=1024, MAX_RECV_BUFFER_SIZE 262144, MAX_LISTEN 16, MAX_URI_SIZE 1024, BUF_SIZE 65535
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
svnrevision: 2:9212M
@(#) $Id: main.c 8772 2012-03-08 11:16:13Z bogdan_iancu $
main.c compiled on 07:29:16 Aug 15 2012 with gcc 4.4.5

Discussion

  • Vladut-Stefan Paiu

    Hello,

    Seems you are having some sort of memory corruption there, related to the dialog module.
    Could you please update to 1.8.2 ? It should contain many fixes compared to 1.8.0.

    If the issue still replicates with the latest 1.8, then let us know.

    Best Regards,
    Vlad

     
  • Bogdan-Andrei Iancu

    • assigned_to: nobody --> vladut-paiu
     
  • Bogdan-Andrei Iancu

    Hi,

    Any update on this Bruce ? between 1.8.0 and 1.8.2 there were several fixes in the "dialog" area, so you problem should be already fixed.

    Regards,
    Bogdan

     

Log in to post a comment.

Get latest updates about Open Source Projects, Conferences and News.

Sign up for the SourceForge newsletter:

JavaScript is required for this form.





No, thanks