run: mdstest 18
Test case failed with "double free" report.
When receive message, mds will go through process to send data to upper layer.
If mds queue ownership is used, message will be put to mailbox through mds_mcm_mailbox_post() and only be read when invoke mds_mailbox_proc().
After put message to mailbox, the send-data process is considered done, mds will delete the buffer previously allocated. This delete is wrong as latter, when message is invoke through mds_mailbox_proc, the receiver will read (invalid read) and try to free the message once again, causing "double free" error.
backtrace:
Thread 1 (Thread 0x7fa902c5bd40 (LWP 694)):
#0 GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
set = {val = {0, 0, 0, 0, 140363863464240, 140363863369568, 3472368028161671168, 0, 0, 206158430216, 140723921026448, 140723921026256, 0, 0, 0, 0}}
pid = <optimized out="">
tid = <optimized out="">
ret = <optimized out=""></optimized></optimized></optimized>
save_stage = 1
act = {sigaction_handler = {sa_handler = 0x0, sa_sigaction = 0x0}, sa_mask = {val = {0 <repeats 14 times>, 140723921025600, 140723921025888}}, sa_flags = -682427840, sa_restorer = 0x1000}
sigs = {val = {32, 0 <repeats 15 times>}}
cnt = <optimized out>
set = <optimized out>
cnt = <optimized out>
set = <optimized out>
ap = {{gp_offset = 24, fp_offset = 32681, overflow_arg_area = 0x7ffcd752fb70, reg_save_area = 0x7ffcd752fb00}}
fd = <optimized out>
list = <optimized out>
nlist = <optimized out>
cp = <optimized out>
written = <optimized out>
on_2 = <optimized out>
next = <optimized out>
str = <optimized out>
len = <optimized out>
newp = <optimized out>
iov = <optimized out>
total = <optimized out>
cnt = <optimized out>
buf = <optimized out>
wp = <optimized out>
old = <optimized out>
cnt = <optimized out>
result = <optimized out>
No locals.
tmp = <optimized out>
tmp = <optimized out>
e = <optimized out>
e = <optimized out>
tc_idx = <optimized out>
tc_idx = <optimized out>
fb = <optimized out>
nextsize = <optimized out>
nextinuse = <optimized out>
prevsize = <optimized out>
fwd = <optimized out>
size = <optimized out>
nextchunk = <optimized out>
bck = <optimized out>
size = <optimized out>
fb = <optimized out>
nextchunk = <optimized out>
nextsize = <optimized out>
nextinuse = <optimized out>
prevsize = <optimized out>
bck = <optimized out>
fwd = <optimized out>
tc_idx = <optimized out>
e = <optimized out>
tmp = <optimized out>
idx = <optimized out>
old = <optimized out>
old2 = <optimized out>
fail = <optimized out>
ignore1 = <optimized out>
ignore2 = <optimized out>
ignore3 = <optimized out>
ignore = <optimized out>
atg1_result = <optimized out>
ret = <optimized out>
ret = <optimized out>
ret = <optimized out>
ret = <optimized out>
ignore1 = <optimized out>
ignore2 = <optimized out>
ignore3 = <optimized out>
heap = <optimized out>
ignore = <optimized out>
ar_ptr = 0x7fa8f4000020
p = 0x7fa8f4001f50
hook = <optimized out>
mem = 0x7fa8f4001f60
ar_ptr = <optimized out>
p = <optimized out>
hook = <optimized out>
x = <optimized out>
ar_ptr = <optimized out>
p = <optimized out>
hook = <optimized out>
x = <optimized out>
No locals.
No locals.
status = 1
cbinfo = {i_yr_svc_hdl = 0, i_yr_svc_id = 512, i_op = MDS_CALLBACK_DIRECT_RECEIVE, info = {cpy = {i_msg = 0x7fa8f4001f60, i_last = 15, i_to_svc_id = 0, o_cpy = 0x0, i_rem_svc_pvt_ver = 200 '\310', o_msg_fmt_ver = 0}, enc = {i_msg = 0x7fa8f4001f60, i_to_svc_id = 15, io_uba = 0x0, i_rem_svc_pvt_ver = 200 '\310', o_msg_fmt_ver = 0}, dec = {io_uba = 0x7fa8f4001f60, i_fr_svc_id = 15, i_is_resp = false, o_msg = 0x0, i_node_id = 200, i_msg_fmt_ver = 0, i_node_name = "\000\000\000\001\000\000\000\000\000\000\266\002\000\000\017\001\002\000\266\002\000\000\017\001\002\000\000\002\000\000\001\000\000\000\017\001\002", '\000' <repeats 217 times>}, enc_flat = {i_msg = 0x7fa8f4001f60, i_to_svc_id = 15, io_uba = 0x0, i_rem_svc_pvt_ver = 200 '\310', o_msg_fmt_ver = 0}, dec_flat = {io_uba = 0x7fa8f4001f60, i_fr_svc_id = 15, i_is_resp = false, o_msg = 0x0, i_node_id = 200, i_msg_fmt_ver = 0, i_node_name = "\000\000\000\001\000\000\000\000\000\000\266\002\000\000\017\001\002\000\266\002\000\000\017\001\002\000\000\002\000\000\001\000\000\000\017\001\002", '\000' <repeats 217 times>}, receive = {i_msg = 0x7fa8f4001f60, i_rsp_reqd = 15, i_msg_ctxt = {length = 0 '\000', data = '\000' <repeats 11 times>}, i_fr_dest = 200, i_fr_svc_id = 256, i_fr_anc = 564113889559222, i_to_dest = 564113889559222, i_to_svc_id = 512, i_priority = MDS_SEND_PRIORITY_LOW, i_node_id = 131343, i_node_name = '\000' <repeats 254 times>, sender_pwe_hdl = 0, i_msg_fmt_ver = 1, pid = 0, uid = 0, gid = 0}, direct_receive = {i_direct_buff = 0x7fa8f4001f60 "\200\362UŽU", i_direct_buff_len = 15, i_rsp_reqd = false, i_msg_ctxt = {length = 0 '\000', data = '\000' <repeats 11 times>}, i_fr_dest = 200, i_fr_svc_id = 256, i_fr_anc = 564113889559222, i_to_dest = 564113889559222, i_to_svc_id = 512, i_priority = MDS_SEND_PRIORITY_LOW, i_node_id = 131343, i_node_name = '\000' <repeats 254 times>, sender_pwe_hdl = 0, i_msg_fmt_ver = 1}, svc_evt = {i_change = 4093648736, i_dest = 15, i_anc = 0, i_role = 200, i_node_id = 0, i_pwe_id = 256, i_svc_id = 0, i_your_id = 694, svc_pwe_hdl = 131343, i_rem_svc_pvt_ver = 182 '\266', i_dest_details = "\002\000\000\017\001\002\000\000\002\000\000\001\000\000\000\017\001\002", '\000' <repeats 261 times>...}, sys_evt = {i_change = 4093648736, i_node_id = 32680, i_evt_mask = 15}, quiesced_ack = {i_dummy = 4093648736}, node_evt = {node_chg = (unknown: 4093648736), node_id = 32680, addr_family = 15, length = 0, ip_addr_len = 0, ip_addr = "\000\000\000\000\000\000\000\000\000\000\310\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\266\002\000\000\017\001\002\000\266\002\000\000\017\001\002\000\000\002\000", i_node_name_len = 1, i_node_name = "\000\000\017\001\002", '\000' <repeats 249 times>}, msg_loss_evt = {i_dest = 140363624882016, i_pwe_id = 15, i_svc_id = 0, i_vdest_id = 0}}}
svc_id = 512
svc_hdl = 562945658454528**
', data = "'7\002\251\177\000\000\240\342\066\002\251"}}, sndack = {itodest = 140363859832577, itimetowait = 100}, rsp = {isenderdest = 140363859832577, imsgctxt = {length = 100 'd', data = "\000\000\000\000\000\000\000
'7\002\251"}}, red = {itovdest = 140363859832577, itoanc = 100}, redrsp = {itovdest = 140363859832577, itoanc = 100, itimetowait = 140363863369568, orsp = 0x7fa90236e2a0 <iofilejumps>, buff = 0xe53e2484eb76700 <error: cannot="" access="" memory="" at="" address="" 0xe53e2484eb76700="">, len = 6096, omsgfmtver = 50108}, redrack = {itovdest = 140363859832577, itoanc = 100, itimetowait = 140363863369568, imsgctxt = {length = 160 '\240', data = "\342\066\002\251\177\000\000\000g\267NH"}}, redack = {itovdest = 140363859832577, itoanc = 100, itimetowait = 140363863369568}, rrsp = {itodest = 140363859832577, itoanc = 100, imsgctxt = {length = 96 '', data = "'7\002\251\177\000\000\240\342\066\002\251"}}, bcast = {ibcastscope = 33632001}, rbcast = {ibcastscope = 33632001}}}, svcdirectsend = {idirectbuff = 0x55bd00000002 <error: Cannot access memory at address 0x55bd00000002>, idirectbufflen = 1832, itosvc = 32681, ipriority = 3310734992, isendtype = 21949, imsgfmtver = 12033, info = {snd = {itodest = 100}, sndrsp = {itodest = 100, itimetowait = 140363863369568, orsp = 0x7fa90236e2a0 <IOfilejumps>, buff = 0xe53e2484eb76700 <error: Cannot access memory at address 0xe53e2484eb76700>, len = 6096, omsgfmtver = 50108}, sndrack = {isenderdest = 100, itimetowait = 140363863369568, imsgctxt = {length = 160 '\240', data = "\342\066\002\251\177\000\000\000g\267NH"}}, sndack = {itodest = 100, itimetowait = 140363863369568}, rsp = {isenderdest = 100, imsgctxt = {length = 96 '
', data = "'7\002\251\177\000\000\240\342\066\002\251"}}, red = {itovdest = 100, itoanc = 140363863369568}, redrsp = {itovdest = 100, itoanc = 140363863369568, itimetowait = 140363863351968, orsp = 0xe53e2484eb76700, buff = 0x55bdc3bc17d0 <gltetvdest+272> "d", len = 26368, omsgfmtver = 20151}, redrack = {itovdest = 100, itoanc = 140363863369568, itimetowait = 140363863351968, imsgctxt = {length = 0 '\000', data = "g\267NH\342S\016\320\027\274ý"}}, redack = {itovdest = 100, itoanc = 140363863369568, itimetowait = 140363863351968}, rrsp = {itodest = 100, itoanc = 140363863369568, imsgctxt = {length = 160 '\240', data = "\342\066\002\251\177\000\000\000g\267NH"}}, bcast = {ibcastscope = 100}, rbcast = {ibcastscope = 100}}}, retrievemsg = {idispatchFlags = SADISPATCHALL}, chgrole = {newrole = VDESTRLSTANDBY}, querydest = {idest = 94270237179906, isvcid = 33621800, iqueryforrole = 169, info = {queryforanc = {ivdestrl = 3310734992, oanc = 140363859832577}, queryforrole = {ianc = 94273547914896, ovdestrl = 33632001}}, olocal = 100, onodeid = 0, oadest = 140363863369568}, querypwe = {opweid = 2, oabsolute = false, info = {absinfo = {oadest = 140363859822376}, virtinfo = {ovdest = 140363859822376, oanc = 94273547914896, orole = 33632001}}}, subscribenode = {idummy = 2}, unsubscribenode = {idummy = 2}}}</gltetvdest+272></error:></iofilejumps></iofilejumps></io21stdout></ionewfilesync+184></ionewfilesync+184></ionewfilesync+184>
Diff:
Diff:
For more information, this ticket is related to ticket-3331, the commit: b65c0887f7d9f240573b7067110cdccb03e79397
Initially, the deallocation added to fix AMF Valgrind report, but deallocate memory in MDS lower layer is not right as it may delete messages before they are read by upper layers. So upper layers must deallocate memory after messages has been read (this point was full-filled from #3331, so remove the wrong memory deallocation in this case will not raise the issue from #3331).
commit 7c5bcb15d96ca1ed042e0403f2971e9ffcbf5da2 (HEAD -> ticket-3337, origin/develop)
Author: dat.tq.phan dat.tq.phan@dektech.com.au
Date: Wed Apr 26 16:31:48 2023 +0700
Last edit: PhanTranQuocDat 2023-05-04