[Openlte-discuss] enb_user_mgr get_instance() lock crash

[Bokath, C. <Chu...@gt...>]

Hello,

I was hoping someone on the list has shared my experienced on the following crash and have been able to track the issue down that occurs roughly every 16 minutes of operations w or w/o users.

I have updated my system per Mikhail Gudkov suggestions on 8/14; 8/11.

I appreciate any insight in advance.

My system crashes with the following output:

terminate called recursively
terminate called after throwing an instance of 'boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::lock_error> >'
  what():  boost: mutex lock failed in pthread_mutex_lock: Invalid argument
Aborted (core dumped)

Back trace of the core file

#0  0x00007fa5951fce37 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
#1  0x00007fa5951fe528 in __GI_abort () at abort.c:89
#2  0x00007fa595b08fed in __gnu_cxx::__verbose_terminate_handler() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#3  0x00007fa595b06ed6 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#4  0x00007fa595b06f21 in std::terminate() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#5  0x00007fa595b07139 in __cxa_throw () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#6  0x000000000042d328 in boost::throw_exception<boost::lock_error> (e=...) at /usr/include/boost/throw_exception.hpp:70
#7  0x000000000042c9c3 in boost::mutex::lock (this=0x81d1c0 <user_mgr_instance_mutex>) at /usr/include/boost/thread/pthread/mutex.hpp:116
#8  0x000000000042ddc7 in boost::unique_lock<boost::mutex>::lock (this=0x7fa562fd7f40) at /usr/include/boost/thread/lock_types.hpp:346
#9  0x000000000042d39d in boost::unique_lock<boost::mutex>::unique_lock (this=0x7fa562fd7f40, m_=...) at /usr/include/boost/thread/lock_types.hpp:124
#10 0x000000000044268f in LTE_fdd_enb_user_mgr::get_instance () at LTE_fdd_enb_user_mgr.cc:79
#11 0x0000000000452577 in LTE_fdd_enb_timer_mgr::handle_tick (this=0x7fa57c000dc0) at LTE_fdd_enb_timer_mgr.cc:223
#12 0x00000000004522dd in LTE_fdd_enb_timer_mgr::handle_msg (this=0x7fa57c000dc0, msg=...) at LTE_fdd_enb_timer_mgr.cc:201
#13 0x00000000004528e5 in LTE_fdd_enb_msgq_cb_wrapper<LTE_fdd_enb_timer_mgr, &LTE_fdd_enb_timer_mgr::handle_msg> (o=0x7fa57c000dc0, msg=...)
    at openlte_v00-19-00/LTE_fdd_enodeb/hdr/LTE_fdd_enb_msgq.h:374
#14 0x000000000043c1c5 in LTE_fdd_enb_msgq_cb::operator() (this=0x7fa57c000b50, msg=...) at LTE_fdd_enb_msgq.cc:79
#15 0x000000000043c809 in LTE_fdd_enb_msgq::receive_thread (inputs=0x7fa57c000b50) at LTE_fdd_enb_msgq.cc:213
#16 0x00007fa59735c0a5 in start_thread (arg=0x7fa562ffd700) at pthread_create.c:309
#17 0x00007fa5952c088d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111


Code:

Frame #10

LTE_fdd_enb_user_mgr* LTE_fdd_enb_user_mgr::get_instance(void)
{
    boost::mutex::scoped_lock lock(user_mgr_instance_mutex);

    if(NULL == instance)
    {
        instance = new LTE_fdd_enb_user_mgr();
    }

    return(instance);
}

Frame #11

void LTE_fdd_enb_timer_mgr::handle_tick(void)
{
    LTE_fdd_enb_user_mgr                            *user_mgr = LTE_fdd_enb_user_mgr::get_instance();
    std::map<uint32, LTE_fdd_enb_timer *>::iterator  iter;
    std::list<uint32>                                expired_list;

    timer_mutex.lock();
    for(iter=timer_map.begin(); iter!=timer_map.end(); iter++)
    {
        (*iter).second->increment();

        if((*iter).second->expired())
        {
            expired_list.push_back((*iter).first);
        }
    }
    timer_mutex.unlock();

    // Delete expired timers
    while(0 != expired_list.size())
    {
        iter = timer_map.find(expired_list.front());
        if(timer_map.end() != iter)
        {
            (*iter).second->call_callback();
            delete (*iter).second;
            timer_map.erase(iter);
        }
        expired_list.pop_front();
    }

    // Pass tick to user_mgr
    user_mgr->handle_tick();
}

Frame #12

void LTE_fdd_enb_timer_mgr::handle_msg(LTE_FDD_ENB_MESSAGE_STRUCT &msg)
{
    if(LTE_FDD_ENB_DEST_LAYER_TIMER_MGR == msg.dest_layer ||
       LTE_FDD_ENB_DEST_LAYER_ANY       == msg.dest_layer)
    {
        switch(msg.type)
        {
        case LTE_FDD_ENB_MESSAGE_TYPE_TIMER_TICK:
            handle_tick();
            break;
        default:
            interface->send_debug_msg(LTE_FDD_ENB_DEBUG_TYPE_WARNING,
                                      LTE_FDD_ENB_DEBUG_LEVEL_TIMER,
                                      __FILE__,
                                      __LINE__,
                                      "Received invalid TIMER message %s",
                                      LTE_fdd_enb_message_type_text[msg.type]);
            break;
        }
    }else{
        interface->send_debug_msg(LTE_FDD_ENB_DEBUG_TYPE_WARNING,
                                  LTE_FDD_ENB_DEBUG_LEVEL_TIMER,
                                  __FILE__,
                                  __LINE__,
                                  "Received message for invalid layer (%s)",
                                  LTE_fdd_enb_dest_layer_text[msg.dest_layer]);
    }
}