OpenSAF / Tickets / #3337 mds: mdstest api coredump when when use MDS queue ownership

Description has changed:

Diff:

--- old
+++ new
@@ -1,33 +1,36 @@
 Steps to reproduce
 ------------------
 run: mdstest 18
+
 Observed behaviour
 ------------------
 Test case failed with &#34;double free&#34; report.
+
 CAUSE:
 -------------------
 When receive message, mds will go through process to send data to upper layer.
 If mds queue ownership is used, message will be put to mailbox through mds_mcm_mailbox_post() and only be read when invoke mds_mailbox_proc().
 After put message to mailbox, the send-data process is considered done, mds will delete the buffer previously allocated. This delete is wrong as latter, when message is invoke through mds_mailbox_proc, the receiver will read (invalid read) and try to free the message once again, causing &#34;double free&#34; error.
+
 Error messages
 ------------------
 backtrace:
- Thread 1 (Thread 0x7f2aa6d2cb00 (LWP 3398)):
+**Thread 1 (Thread 0x7fa902c5bd40 (LWP 694)):
 **#0  GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
-        set = {val = {0, 0, 0, 206158430210, 0, 139821157765124, 562945658454528, 139821157765124, 139821164183276, 139821157854076, 139821164183248, 281470681874431, 131071, 6794816100368768768, 564113889561918, 139821049716696}}
+        set = {val = {0, 0, 0, 0, 140363863464240, 140363863369568, 3472368028161671168, 0, 0, 206158430216, 140723921026448, 140723921026256, 0, 0, 0, 0}}
         pid = &lt;optimized out&gt;
         tid = &lt;optimized out&gt;
         ret = &lt;optimized out&gt;
-#1  0x00007f2aa60fe7f1 in GI_abort () at abort.c:79
+#1  0x00007fa901fc67f1 in GI_abort () at abort.c:79
         save_stage = 1
-        act = {sigaction_handler = {sa_handler = 0xa03bffff, sa_sigaction = 0xa03bffff}, sa_mask = {val = {1, 564113889561918, 139821157846167, 2199023255553, 139821049716586, 4295032831, 564113889561918, 0, 282333970170112, 2, 2563, 3390, 139821164183768, 1, 139821164183744, 139821164184032}}, sa_flags = -1496137536, sa_restorer = 0x1000}
+        act = {sigaction_handler = {sa_handler = 0x0, sa_sigaction = 0x0}, sa_mask = {val = {0 &lt;repeats 14 times&gt;, 140723921025600, 140723921025888}}, sa_flags = -682427840, sa_restorer = 0x1000}
         sigs = {val = {32, 0 &lt;repeats 15 times&gt;}}
         cnt = &lt;optimized out&gt;
         set = &lt;optimized out&gt;
         cnt = &lt;optimized out&gt;
         set = &lt;optimized out&gt;
-#2  0x00007f2aa6147837 in libc_message (action=action@entry=do_abort, fmt=fmt@entry=0x7f2aa6274a7b &#34;%s\n&#34;) at ../sysdeps/posix/libc_fatal.c:181
-        ap = {{gp_offset = 24, fp_offset = 0, overflow_arg_area = 0x7f2aa6d2c1f0, reg_save_area = 0x7f2aa6d2c180}}
+#2  0x00007fa90200f837 in libc_message (action=action@entry=do_abort, fmt=fmt@entry=0x7fa90213ca7b &#34;%s\n&#34;) at ../sysdeps/posix/libc_fatal.c:181
+        ap = {{gp_offset = 24, fp_offset = 32681, overflow_arg_area = 0x7ffcd752fb70, reg_save_area = 0x7ffcd752fb00}}
         fd = &lt;optimized out&gt;
         list = &lt;optimized out&gt;
         nlist = &lt;optimized out&gt;
@@ -46,15 +49,15 @@
         old = &lt;optimized out&gt;
         cnt = &lt;optimized out&gt;
         result = &lt;optimized out&gt;
-#3  0x00007f2aa614e8ba in malloc_printerr (str=str@entry=0x7f2aa6276740 &#34;double free or corruption (fasttop)&#34;) at malloc.c:5342
+#3  0x00007fa9020168ba in malloc_printerr (str=str@entry=0x7fa90213e6e8 &#34;free(): double free detected in tcache 2&#34;) at malloc.c:5342
 No locals.
-#4  0x00007f2aa6259c4b in _int_free (have_lock=0, p=0x7f2aa0002160, av=0x7f2aa0000020) at malloc.c:4260
-        idx = &lt;optimized out&gt;
-        old = &lt;optimized out&gt;
-        idx = &lt;optimized out&gt;
-        old = &lt;optimized out&gt;
-        old2 = &lt;optimized out&gt;
-        old2 = &lt;optimized out&gt;
+#4  0x00007fa90201e0ed in _int_free (have_lock=0, p=0x7fa8f4001f50, av=0x7fa8f4000020) at malloc.c:4195
+        tmp = &lt;optimized out&gt;
+        tmp = &lt;optimized out&gt;
+        e = &lt;optimized out&gt;
+        e = &lt;optimized out&gt;
+        tc_idx = &lt;optimized out&gt;
+        tc_idx = &lt;optimized out&gt;
         fb = &lt;optimized out&gt;
         nextsize = &lt;optimized out&gt;
         nextinuse = &lt;optimized out&gt;
@@ -92,11 +95,11 @@
         ignore3 = &lt;optimized out&gt;
         heap = &lt;optimized out&gt;
         ignore = &lt;optimized out&gt;
-#5  GI_libc_free (mem=0x7f2aa0002170) at malloc.c:3134
-        ar_ptr = 0x7f2aa0000020
-        p = 0x7f2aa0002160
+#5  GI_libc_free (mem=0x7fa8f4001f60) at malloc.c:3134
+        ar_ptr = 0x7fa8f4000020
+        p = 0x7fa8f4001f50
         hook = &lt;optimized out&gt;
-        mem = 0x7f2aa0002170
+        mem = 0x7fa8f4001f60
         ar_ptr = &lt;optimized out&gt;
         p = &lt;optimized out&gt;
         hook = &lt;optimized out&gt;
@@ -104,27 +107,40 @@
         ar_ptr = &lt;optimized out&gt;
         p = &lt;optimized out&gt;
         hook = &lt;optimized out&gt;
-        ar_ptr = &lt;optimized out&gt;
-        p = &lt;optimized out&gt;
-        hook = &lt;optimized out&gt;
         x = &lt;optimized out&gt;
-#6  tcache_thread_shutdown () at malloc.c:2979
-        e = 0x7f2aa0002170
-        i = &lt;optimized out&gt;
-        tcache_tmp = &lt;optimized out&gt;
-        i = &lt;optimized out&gt;
-        tcache_tmp = &lt;optimized out&gt;
-        e = &lt;optimized out&gt;
-#7  arena_thread_freeres () at arena.c:950
-        a = &lt;optimized out&gt;
-        PRETTY_FUNCTION = &#34;arena_thread_freeres&#34;
-#8  0x00007f2aa625a562 in libc_thread_freeres () at thread-freeres.c:29
-        ptr = 0x7f2aa64a5740 &lt;elf_set_libc_thread_subfreeres_element_arena_thread_freeres&gt;
-#9  0x00007f2aa64b6700 in start_thread (arg=0x7f2aa6d2cb00) at pthread_create.c:476
-        pd = 0x7f2aa6d2cb00
-        now = &lt;optimized out&gt;
-        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139821164186368, -4330361106019401868, 139821164184448, 1, 0, 140731572523136, 4445679699519848308, 4445680733411852148}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
-        not_first_call = &lt;optimized out&gt;
-#10 0x00007f2aa61df61f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
+#6  0x00007fa9025fc2fa in mds_free_direct_buff (buff=&lt;optimized out&gt;) at src/mds/mds_papi.c:336
 No locals.
-51 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.**
+#7  0x000055bdc36727d0 in tet_mds_cb_direct_rcv (mds_to_svc_info=0x7ffcd752fc20) at src/mds/apitest/mdstipc_conf.c:2196
+No locals.
+#8  0x00007fa9025f1671 in mds_mailbox_proc (msgelem=0x7fa8f4002510, svc_cb=svc_cb@entry=0x55bdc555e060) at src/mds/mds_c_sndrcv.c:6991
+        status = 1
+        cbinfo = {i_yr_svc_hdl = 0, i_yr_svc_id = 512, i_op = MDS_CALLBACK_DIRECT_RECEIVE, info = {cpy = {i_msg = 0x7fa8f4001f60, i_last = 15, i_to_svc_id = 0, o_cpy = 0x0, i_rem_svc_pvt_ver = 200 &#39;\310&#39;, o_msg_fmt_ver = 0}, enc = {i_msg = 0x7fa8f4001f60, i_to_svc_id = 15, io_uba = 0x0, i_rem_svc_pvt_ver = 200 &#39;\310&#39;, o_msg_fmt_ver = 0}, dec = {io_uba = 0x7fa8f4001f60, i_fr_svc_id = 15, i_is_resp = false, o_msg = 0x0, i_node_id = 200, i_msg_fmt_ver = 0, i_node_name = &#34;\000\000\000\001\000\000\000\000\000\000\266\002\000\000\017\001\002\000\266\002\000\000\017\001\002\000\000\002\000\000\001\000\000\000\017\001\002&#34;, &#39;\000&#39; &lt;repeats 217 times&gt;}, enc_flat = {i_msg = 0x7fa8f4001f60, i_to_svc_id = 15, io_uba = 0x0, i_rem_svc_pvt_ver = 200 &#39;\310&#39;, o_msg_fmt_ver = 0}, dec_flat = {io_uba = 0x7fa8f4001f60, i_fr_svc_id = 15, i_is_resp = false, o_msg = 0x0, i_node_id = 200, i_msg_fmt_ver = 0, i_node_name = &#34;\000\000\000\001\000\000\000\000\000\000\266\002\000\000\017\001\002\000\266\002\000\000\017\001\002\000\000\002\000\000\001\000\000\000\017\001\002&#34;, &#39;\000&#39; &lt;repeats 217 times&gt;}, receive = {i_msg = 0x7fa8f4001f60, i_rsp_reqd = 15, i_msg_ctxt = {length = 0 &#39;\000&#39;, data = &#39;\000&#39; &lt;repeats 11 times&gt;}, i_fr_dest = 200, i_fr_svc_id = 256, i_fr_anc = 564113889559222, i_to_dest = 564113889559222, i_to_svc_id = 512, i_priority = MDS_SEND_PRIORITY_LOW, i_node_id = 131343, i_node_name = &#39;\000&#39; &lt;repeats 254 times&gt;, sender_pwe_hdl = 0, i_msg_fmt_ver = 1, pid = 0, uid = 0, gid = 0}, direct_receive = {i_direct_buff = 0x7fa8f4001f60 &#34;\200\362UŽU&#34;, i_direct_buff_len = 15, i_rsp_reqd = false, i_msg_ctxt = {length = 0 &#39;\000&#39;, data = &#39;\000&#39; &lt;repeats 11 times&gt;}, i_fr_dest = 200, i_fr_svc_id = 256, i_fr_anc = 564113889559222, i_to_dest = 564113889559222, i_to_svc_id = 512, i_priority = MDS_SEND_PRIORITY_LOW, i_node_id = 131343, i_node_name = &#39;\000&#39; &lt;repeats 254 times&gt;, sender_pwe_hdl = 0, i_msg_fmt_ver = 1}, svc_evt = {i_change = 4093648736, i_dest = 15, i_anc = 0, i_role = 200, i_node_id = 0, i_pwe_id = 256, i_svc_id = 0, i_your_id = 694, svc_pwe_hdl = 131343, i_rem_svc_pvt_ver = 182 &#39;\266&#39;, i_dest_details = &#34;\002\000\000\017\001\002\000\000\002\000\000\001\000\000\000\017\001\002&#34;, &#39;\000&#39; &lt;repeats 261 times&gt;...}, sys_evt = {i_change = 4093648736, i_node_id = 32680, i_evt_mask = 15}, quiesced_ack = {i_dummy = 4093648736}, node_evt = {node_chg = (unknown: 4093648736), node_id = 32680, addr_family = 15, length = 0, ip_addr_len = 0, ip_addr = &#34;\000\000\000\000\000\000\000\000\000\000\310\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\266\002\000\000\017\001\002\000\266\002\000\000\017\001\002\000\000\002\000&#34;, i_node_name_len = 1, i_node_name = &#34;\000\000\017\001\002&#34;, &#39;\000&#39; &lt;repeats 249 times&gt;}, msg_loss_evt = {i_dest = 140363624882016, i_pwe_id = 15, i_svc_id = 0, i_vdest_id = 0}}}
+        svc_id = 512
+        svc_hdl = 562945658454528**
+##         localcbptr = 0x55bdc3672d20 &lt;tetmdssvccallback&gt;
+###9  0x00007fa9025f1adb in mdsretrieve (info=info@entry=0x7ffcd752fe70) at src/mds/mdscsndrcv.c:6732
+##         svcid = 512
+##         localmbx = 4290772993
+##         msgelem = &lt;optimized out&gt;
+##         hdl = 0x55bdc555e060
+##         svccb = 0x55bdc555e060
+###10 0x00007fa9025fc0a8 in ncsmdsapi (svctomdsinfo=svctomdsinfo@entry=0x7ffcd752fe70) at src/mds/mdspapi.c:169
+##         status = &lt;optimized out&gt;
+###11 0x000055bdc3671ed5 in mdsserviceretrieve (mdshdl=&lt;optimized out&gt;, svcid=svcid@entry=512, dispatchFlags=dispatchFlags@entry=SADISPATCHALL) at src/mds/apitest/mdstipcconf.c:1765
+##         svctomdsinfo = {imdshdl = 131071, isvcid = 512, iop = MDSRETRIEVE, info = {svcinstall = {iyrsvchdl = 94270237179906, iinstallscope = 33621800, isvccb = 0x55bdc555ce90, odest = 140363859832577, oanc = 100, imdsqownership = 96, oselobj = {raiseobj = 32681, rmvobj = 37151392}, imdssvcpvtver = 169 &#39;\251&#39;, ifailnoactivesends = 127, imsglossindication = false}, svcuninstall = {imsgfreecb = 0x55bd00000002}, svcsubscribe = {iscope = NCSMDSSCOPEINTRANODE, inumsvcs = 189 &#39;\275&#39;, isvcids = 0x7fa902010728 &lt;IOnewfilesync+184&gt;}, redsubscribe = {iscope = NCSMDSSCOPEINTRANODE, inumsvcs = 189 &#39;\275&#39;, isvcids = 0x7fa902010728 &lt;IOnewfilesync+184&gt;}, svccancel = {inumsvcs = 2 &#39;\002&#39;, isvcids = 0x7fa902010728 &lt;IOnewfilesync+184&gt;}, svcsyssubscribe = {ievtmap = 2}, svcsend = {imsg = 0x55bd00000002, itosvc = 33621800, ipriority = 32681, isendtype = 3310734992, info = {snd = {itodest = 140363859832577}, sndrsp = {itodest = 140363859832577, itimetowait = 100, orsp = 0x7fa902372760 &lt;IO21stdout&gt;, buff = 0x7fa90236e2a0 &lt;IOfilejumps&gt; &#34;&#34;, len = 26368, omsgfmtver = 20151}, sndrack = {isenderdest = 140363859832577, itimetowait = 100, imsgctxt = {length = 96 &#39;`&#39;, data = &#34;&#39;7\002\251\177\000\000\240\342\066\002\251&#34;}}, sndack = {itodest = 140363859832577, itimetowait = 100}, rsp = {isenderdest = 140363859832577, imsgctxt = {length = 100 &#39;d&#39;, data = &#34;\000\000\000\000\000\000\000`&#39;7\002\251&#34;}}, red = {itovdest = 140363859832577, itoanc = 100}, redrsp = {itovdest = 140363859832577, itoanc = 100, itimetowait = 140363863369568, orsp = 0x7fa90236e2a0 &lt;IOfilejumps&gt;, buff = 0xe53e2484eb76700 &lt;error: Cannot access memory at address 0xe53e2484eb76700&gt;, len = 6096, omsgfmtver = 50108}, redrack = {itovdest = 140363859832577, itoanc = 100, itimetowait = 140363863369568, imsgctxt = {length = 160 &#39;\240&#39;, data = &#34;\342\066\002\251\177\000\000\000g\267NH&#34;}}, redack = {itovdest = 140363859832577, itoanc = 100, itimetowait = 140363863369568}, rrsp = {itodest = 140363859832577, itoanc = 100, imsgctxt = {length = 96 &#39;`&#39;, data = &#34;&#39;7\002\251\177\000\000\240\342\066\002\251&#34;}}, bcast = {ibcastscope = 33632001}, rbcast = {ibcastscope = 33632001}}}, svcdirectsend = {idirectbuff = 0x55bd00000002 &lt;error: Cannot access memory at address 0x55bd00000002&gt;, idirectbufflen = 1832, itosvc = 32681, ipriority = 3310734992, isendtype = 21949, imsgfmtver = 12033, info = {snd = {itodest = 100}, sndrsp = {itodest = 100, itimetowait = 140363863369568, orsp = 0x7fa90236e2a0 &lt;IOfilejumps&gt;, buff = 0xe53e2484eb76700 &lt;error: Cannot access memory at address 0xe53e2484eb76700&gt;, len = 6096, omsgfmtver = 50108}, sndrack = {isenderdest = 100, itimetowait = 140363863369568, imsgctxt = {length = 160 &#39;\240&#39;, data = &#34;\342\066\002\251\177\000\000\000g\267NH&#34;}}, sndack = {itodest = 100, itimetowait = 140363863369568}, rsp = {isenderdest = 100, imsgctxt = {length = 96 &#39;`&#39;, data = &#34;&#39;7\002\251\177\000\000\240\342\066\002\251&#34;}}, red = {itovdest = 100, itoanc = 140363863369568}, redrsp = {itovdest = 100, itoanc = 140363863369568, itimetowait = 140363863351968, orsp = 0xe53e2484eb76700, buff = 0x55bdc3bc17d0 &lt;gltetvdest+272&gt; &#34;d&#34;, len = 26368, omsgfmtver = 20151}, redrack = {itovdest = 100, itoanc = 140363863369568, itimetowait = 140363863351968, imsgctxt = {length = 0 &#39;\000&#39;, data = &#34;g\267NH\342S\016\320\027\274ý&#34;}}, redack = {itovdest = 100, itoanc = 140363863369568, itimetowait = 140363863351968}, rrsp = {itodest = 100, itoanc = 140363863369568, imsgctxt = {length = 160 &#39;\240&#39;, data = &#34;\342\066\002\251\177\000\000\000g\267NH&#34;}}, bcast = {ibcastscope = 100}, rbcast = {ibcastscope = 100}}}, retrievemsg = {idispatchFlags = SADISPATCHALL}, chgrole = {newrole = VDESTRLSTANDBY}, querydest = {idest = 94270237179906, isvcid = 33621800, iqueryforrole = 169, info = {queryforanc = {ivdestrl = 3310734992, oanc = 140363859832577}, queryforrole = {ianc = 94273547914896, ovdestrl = 33632001}}, olocal = 100, onodeid = 0, oadest = 140363863369568}, querypwe = {opweid = 2, oabsolute = false, info = {absinfo = {oadest = 140363859822376}, virtinfo = {ovdest = 140363859822376, oanc = 94273547914896, orole = 33632001}}}, subscribenode = {idummy = 2}, unsubscribenode = {idummy = 2}}}
+###12 0x000055bdc365b4ad in tetcleanupsetup () at src/mds/apitest/mdstipcapi.c:3339
+##         i = 512
+##         id = &lt;optimized out&gt;
+##         FAIL = 0
+###13 0x000055bdc366a8a1 in tetdirectbroadcasttosvctp6 () at src/mds/apitest/mdstipcapi.c:12780
+##         FAIL = 0
+##         svcids = {512}
+###14 0x000055bdc3672ef9 in runtestcase (suite=&lt;optimized out&gt;, tcase=&lt;optimized out&gt;) at src/osaf/apitest/utest.c:178
+## No locals.
+###15 0x000055bdc367333e in testrun (suite=18, tcase=6) at src/osaf/apitest/utest.c:226
+##         i = &lt;optimized out&gt;
+##         j = &lt;optimized out&gt;
+###16 0x000055bdc3650859 in main (argc=3, argv=0x7ffcd75300c8) at src/mds/apitest/mdstest.c:92
+##         suite = &lt;optimized out&gt;
+##         tcase = &lt;optimized out&gt;
+##         rc = &lt;optimized out&gt;***

Attachments has changed:

Diff:

--- old
+++ new
@@ -0,0 +1 @@
+bt_core.1682494999.mdstest.694.SC-1 (16.0 kB; application/octet-stream)

mds: mdstest api coredump when when use MDS queue ownership

Milestone

Searches

Help

#3337 mds: mdstest api coredump when when use MDS queue ownership

Steps to reproduce

Observed behaviour

CAUSE:

Error messages

1 0x00007fa901fc67f1 in GI_abort () at abort.c:79

2 0x00007fa90200f837 in libc_message (action=action@entry=do_abort, fmt=fmt@entry=0x7fa90213ca7b "%s\n") at ../sysdeps/posix/libc_fatal.c:181

3 0x00007fa9020168ba in malloc_printerr (str=str@entry=0x7fa90213e6e8 "free(): double free detected in tcache 2") at malloc.c:5342

4 0x00007fa90201e0ed in _int_free (have_lock=0, p=0x7fa8f4001f50, av=0x7fa8f4000020) at malloc.c:4195

5 GI_libc_free (mem=0x7fa8f4001f60) at malloc.c:3134

6 0x00007fa9025fc2fa in mds_free_direct_buff (buff=<optimized out="">) at src/mds/mds_papi.c:336</optimized>

7 0x000055bdc36727d0 in tet_mds_cb_direct_rcv (mds_to_svc_info=0x7ffcd752fc20) at src/mds/apitest/mdstipc_conf.c:2196

8 0x00007fa9025f1671 in mds_mailbox_proc (msgelem=0x7fa8f4002510, svc_cb=svc_cb@entry=0x55bdc555e060) at src/mds/mds_c_sndrcv.c:6991

localcbptr = 0x55bdc3672d20 <tetmdssvccallback></tetmdssvccallback>

9 0x00007fa9025f1adb in mdsretrieve (info=info@entry=0x7ffcd752fe70) at src/mds/mdscsndrcv.c:6732

svcid = 512

localmbx = 4290772993

msgelem = <optimized out=""></optimized>

hdl = 0x55bdc555e060

svccb = 0x55bdc555e060

10 0x00007fa9025fc0a8 in ncsmdsapi (svctomdsinfo=svctomdsinfo@entry=0x7ffcd752fe70) at src/mds/mdspapi.c:169

status = <optimized out=""></optimized>

11 0x000055bdc3671ed5 in mdsserviceretrieve (mdshdl=<optimized out="">, svcid=svcid@entry=512, dispatchFlags=dispatchFlags@entry=SADISPATCHALL) at src/mds/apitest/mdstipcconf.c:1765</optimized>

12 0x000055bdc365b4ad in tetcleanupsetup () at src/mds/apitest/mdstipcapi.c:3339

i = 512

id = <optimized out=""></optimized>

FAIL = 0

13 0x000055bdc366a8a1 in tetdirectbroadcasttosvctp6 () at src/mds/apitest/mdstipcapi.c:12780

FAIL = 0

svcids = {512}

14 0x000055bdc3672ef9 in runtestcase (suite=<optimized out="">, tcase=<optimized out="">) at src/osaf/apitest/utest.c:178</optimized></optimized>

No locals.

15 0x000055bdc367333e in testrun (suite=18, tcase=6) at src/osaf/apitest/utest.c:226

i = <optimized out=""></optimized>

j = <optimized out=""></optimized>

16 0x000055bdc3650859 in main (argc=3, argv=0x7ffcd75300c8) at src/mds/apitest/mdstest.c:92

suite = <optimized out=""></optimized>

tcase = <optimized out=""></optimized>

rc = <optimized out="">***</optimized>

Related

Discussion