#2345 amf: standby controller reboots after switchover

[Nagendra Kumar]

Steps to reproduce

1. Make the following changes at Standby Amfd (SC-2) :
   diff --git a/src/amf/amfnd/evt.cc b/src/amf/amfnd/evt.cc
   --- a/src/amf/amfnd/evt.cc
   +++ b/src/amf/amfnd/evt.cc
   @@ -71,6 +71,11 @@ AVND_EVT avnd_evt_create(AVND_CB cb,
   / fill the event specific fields /
   switch (type) {
   / AvD event types /
2. case AVND_EVT_AVD_ROLE_CHANGE_MSG:
3. case AVND_EVT_AVD_VERIFY_MSG:
4. evt->priority = NCS_IPC_PRIORITY_VERY_HIGH; / bump up the priority /
5. evt->info.avd = (AVSV_DND_MSG *)info;
6. break;
   case AVND_EVT_AVD_NODE_UP_MSG:
   case AVND_EVT_AVD_REG_SU_MSG:
   case AVND_EVT_AVD_REG_COMP_MSG:
   @@ -79,12 +84,10 @@ AVND_EVT avnd_evt_create(AVND_CB cb,
   case AVND_EVT_AVD_PG_UPD_MSG:
   case AVND_EVT_AVD_OPERATION_REQUEST_MSG:
   case AVND_EVT_AVD_SU_PRES_MSG:
7. case AVND_EVT_AVD_VERIFY_MSG:
   case AVND_EVT_AVD_ACK_MSG:
   case AVND_EVT_AVD_SHUTDOWN_APP_SU_MSG:
   case AVND_EVT_AVD_SET_LEDS_MSG:
   case AVND_EVT_AVD_COMP_VALIDATION_RESP_MSG:
8. case AVND_EVT_AVD_ROLE_CHANGE_MSG:
   case AVND_EVT_AVD_ADMIN_OP_REQ_MSG:
   case AVND_EVT_AVD_REBOOT_MSG:
   case AVND_EVT_AVD_COMPCSI_ASSIGN_MSG:
   diff --git a/src/amf/amfnd/mds.cc b/src/amf/amfnd/mds.cc
   --- a/src/amf/amfnd/mds.cc
   +++ b/src/amf/amfnd/mds.cc
   @@ -543,6 +543,9 @@ uint32_t avnd_mds_svc_evt(AVND_CB *cb, M
   // DOWN is received for the old director ..
   if (m_AVND_CB_IS_AVD_UP(cb)) {
   m_AVND_CB_AVD_UP_RESET(cb);
9. LOG_NO("Before sleep");
10. sleep(2);

11. LOG_NO("After sleep");
    }

                    evt = avnd_evt_create(cb, AVND_EVT_MDS_AVD_UP, 0, &evt_info->i_dest, 0, 0, 0);
    

12. Start both controllers. Upload app configuration, hosting SC-1 on SC-2.

13. Issue amf-adm si-swap safSi=SC-2N,safApp=OpenSAF
14. Then issue amf-adm unlock-in safSu=SU1,safSg=AmfDemo_2N,safApp=AmfDemo1
    Observed behaviour

---

Mar 6 12:53:58 PM_SC-2 osafamfnd[8501]: NO After sleep
Mar 6 12:54:01 PM_SC-2 osafamfnd[8501]: Rebooting OpenSAF NodeId = 0 EE Name = No EE Mapped, Reason: Message ID mismatch, rec 2, expected 1, OwnNodeId = 131599, SupervisionTime = 60
Mar 6 12:54:01 PM_SC-2 opensaf_reboot: Rebooting local node; timeout=60

This happens because role change(avnd_evt_mds_avd_up_evh) comes before avnd_evt_mds_avd_up_evh.

osafamfnd [9054:src/amf/amfnd/mds.cc:0540] NO AVD NEW_ACTIVE, adest:1
osafamfnd [9054:src/amf/amfnd/mds.cc:0546] NO Before sleep
osafamfnd [9054:src/amf/amfnd/mds.cc:0548] NO After sleep
osafamfnd [9054:src/amf/amfnd/mds.cc:0345] T1 Active AVD Adest = 565214626185244
osafamfnd [9054:src/amf/amfnd/main.cc:0647] >> avnd_evt_process
osafamfnd [9054:src/amf/amfnd/main.cc:0664] TR Evt type:9
osafamfnd [9054:src/amf/amfnd/verify.cc:0058] >> avnd_evt_avd_verify_evh: Data Verify message received from newly ACTIVE AVD
osafamfnd [9054:src/amf/amfnd/verify.cc:0071] T1 AVD send ID count: 7
osafamfnd [9054:src/amf/amfnd/verify.cc:0072] T1 AVND receive ID count: 7
osafamfnd [9054:src/amf/amfnd/di.cc:1087] >> avnd_di_ack_nack_msg_send: Receive id = 7
osafamfnd [9054:src/amf/amfnd/di.cc:1103] T1 MsgId=84,ACK=1
osafamfnd [9054:src/amf/amfnd/di.cc:1033] >> avnd_di_msg_send: Msg type '10'
osafamfnd [9054:src/amf/amfnd/di.cc:1043] T1 avnd_di_msg_send, Active AVD Adest: 565214626185244
osafamfnd [9054:src/amf/amfnd/mds.cc:1496] >> avnd_mds_red_send: Msg type '1'
osafamfnd [9054:src/amf/amfnd/mds.cc:1534] << avnd_mds_red_send: rc '1'
osafamfnd [9054:src/amf/amfnd/di.cc:1065] << avnd_di_msg_send: 1
osafamfnd [9054:src/amf/amfnd/di.cc:1112] << avnd_di_ack_nack_msg_send: retval=1
osafamfnd [9054:src/amf/amfnd/verify.cc:0095] T1 AVD receive ID count: 83
osafamfnd [9054:src/amf/amfnd/verify.cc:0096] T1 AVND send ID count: 83
osafamfnd [9054:src/amf/amfnd/di.cc:1571] >> avnd_di_resend_pg_start_track
osafamfnd [9054:src/amf/amfnd/di.cc:1581] << avnd_di_resend_pg_start_track
osafamfnd [9054:src/amf/amfnd/verify.cc:0143] << avnd_evt_avd_verify_evh
osafamfnd [9054:src/amf/amfnd/main.cc:0670] TR Evt Type:9 success
osafamfnd [9054:src/amf/amfnd/main.cc:0675] << avnd_evt_process
osafamfnd [9054:src/amf/amfnd/main.cc:0647] >> avnd_evt_process
osafamfnd [9054:src/amf/amfnd/main.cc:0664] TR Evt type:14
osafamfnd [9054:src/amf/amfnd/di.cc:1503] >> avnd_evt_avd_role_change_evh
osafamfnd [9054:src/amf/amfnd/di.cc:1507] IN AVD is not up yet
osafamfnd [9054:src/amf/amfnd/di.cc:0000] << avnd_evt_avd_role_change_evh
osafamfnd [9054:src/amf/amfnd/main.cc:0670] TR Evt Type:14 failure
osafamfnd [9054:src/amf/amfnd/main.cc:0675] << avnd_evt_process
osafamfnd [9054:src/amf/amfnd/main.cc:0647] >> avnd_evt_process
osafamfnd [9054:src/amf/amfnd/main.cc:0664] TR Evt type:44
osafamfnd [9054:src/amf/amfnd/di.cc:0510] >> avnd_evt_mds_avd_up_evh: 1
osafamfnd [9054:src/amf/amfnd/di.cc:0580] << avnd_evt_mds_avd_up_evh
osafamfnd [9054:src/amf/amfnd/main.cc:0670] TR Evt Type:44 success
osafamfnd [9054:src/amf/amfnd/main.cc:0675] << avnd_evt_process
osafamfnd [9054:src/amf/amfnd/main.cc:0647] >> avnd_evt_process

[Nagendra Kumar]

changeset: 8651:a90faf589254
branch: opensaf-5.0.x
parent: 8642:678a5e2a2895
user: Nagendra Kumarnagendra.k@oracle.com
date: Tue Mar 07 11:02:10 2017 +0530
summary: amfnd: update counters during switchover [#2345]

changeset: 8652:a7c62f1de1a3
branch: opensaf-5.1.x
parent: 8643:64e57b58e63a
user: Nagendra Kumarnagendra.k@oracle.com
date: Tue Mar 07 11:02:23 2017 +0530
summary: amfnd: update counters during switchover [#2345]

changeset: 8653:ce11c226f63a
tag: tip
parent: 8650:b1794d3c341d
user: Nagendra Kumarnagendra.k@oracle.com
date: Tue Mar 07 11:05:23 2017 +0530
summary: amfnd: update counters during switchover [#2345]

[staging:a90faf]
[staging:a7c62f]
[staging:ce11c2]