Menu
▾
▴
[devel] [PATCH 1 of 2] amfnd: fix continuous restart of a inst_failed comp during su restart[#1556]
|
From: <pra...@or...> - 2016-01-08 13:27:25
|
osaf/services/saf/amf/amfnd/clc.cc | 87 +++++++++++++++++++-------
osaf/services/saf/amf/amfnd/include/avnd_su.h | 2 +
osaf/services/saf/amf/amfnd/su.cc | 18 +++++
osaf/services/saf/amf/amfnd/susm.cc | 19 +++++-
4 files changed, 100 insertions(+), 26 deletions(-)
During su-restart recovery or RESTART admin op, if instantiation of a component
fails amfnd keeps on restarting it after each inst_failure.
AMFND must give up re-instantiation of a inst_fail compoent after trying
saAmfNumMaxInstantiateWithoutDelay value of comp global class or
saAmfCompNumMaxInstantiateWithoutDelay value in comp class.
Patch stops re-instantiation of a inst_failed comp when max retries is
reached.
diff --git a/osaf/services/saf/amf/amfnd/clc.cc b/osaf/services/saf/amf/amfnd/clc.cc
--- a/osaf/services/saf/amf/amfnd/clc.cc
+++ b/osaf/services/saf/amf/amfnd/clc.cc
@@ -891,7 +891,8 @@ uint32_t avnd_comp_clc_fsm_run(AVND_CB *
(final_st == SA_AMF_PRESENCE_RESTARTING)) &&
((ev == AVND_COMP_CLC_PRES_FSM_EV_INST_SUCC) ||
(ev == AVND_COMP_CLC_PRES_FSM_EV_TERM_SUCC) ||
- (ev == AVND_COMP_CLC_PRES_FSM_EV_CLEANUP_SUCC))))
+ (ev == AVND_COMP_CLC_PRES_FSM_EV_CLEANUP_SUCC)) &&
+ (comp->clc_info.exec_cmd == AVND_COMP_CLC_CMD_TYPE_NONE)))
rc = avnd_comp_clc_st_chng_prc(cb, comp, prv_st, final_st);
done:
@@ -1669,6 +1670,62 @@ uint32_t avnd_comp_clc_insting_clean_hdl
return rc;
}
+/**
+ * @brief Checks for eligibility of a failed component for instantiation
+ * when it has been successfully cleaned up. Instantiation depends
+ * upon recovery context or Restart admin op.
+ *
+ * @return true/false
+ */
+static bool is_failed_comp_eligible_for_instantiation(AVND_COMP *comp) {
+
+ if (isRestartSet(comp->su)) { //SU is restarting (RESTART admin op or recovery policy).
+ if (isFailed(comp->su)) { //SU is failed (case surestart recovery).
+ /*During surestart recovery, after cleanup of all components, amfnd starts
+ instantiation of components. A component may fault at this stage. Such a
+ component is eligible for instantiation.*/
+ if ((comp->pres == SA_AMF_PRESENCE_INSTANTIATING) &&
+ (comp->su->pres == SA_AMF_PRESENCE_INSTANTIATING))
+ return true;
+
+ /*This component is being cleaned up because of su restart recovery.
+ In this case component is not eligible for instantiation.*/
+ if ((comp->pres == SA_AMF_PRESENCE_RESTARTING) &&
+ (comp->su->pres == SA_AMF_PRESENCE_TERMINATING)) {
+ /* Component got cleaned up in RESTARTING state and surestart recovery is going on.
+ In surestart recovery component never enters in RESTARTING state. This means
+ component was cleaned up in the context of comp-restart recovery.
+ Since further escalation has reached to surestart, same cleanup can be used
+ and thus comp can be marked uninstantiated.*/
+ avnd_comp_pres_state_set(comp, SA_AMF_PRESENCE_UNINSTANTIATED);
+ return false;
+ }
+ } else { //Case of RESTART admin op or assignment phase of surestart recovery.
+ if (isAdminRestarted(comp->su)) { //RESTART admin op.
+ /*During RESTART admin op on SU, after termination of all components, amfnd
+ starts instantiation of components. A component may fault at this stage.
+ Such a component is eligible for instantiation.*/
+ if ((comp->pres == SA_AMF_PRESENCE_RESTARTING) ||
+ (comp->pres == SA_AMF_PRESENCE_INSTANTIATING))
+ return true;
+ } else {//Assignment phase during RESTART admin op or during surestart recovery.
+ /*After successful instantiation of all the components of SU because of either
+ su restart recovery or RESTART admin op on SU, amfnd starts reassigning
+ the components in SU. A component may fault during reassignment. Such a
+ component is eligible for instantiation.*/
+ return true;
+ }
+ }
+ } else {//SU is not restarting.
+ if (!isFailed(comp->su)) {//SU is not failed.
+ /*A failed component is eligible for instantiation
+ in the context of comp-restart recovery.*/
+ return true;
+ }
+ }
+
+ return false;
+}
/****************************************************************************
Name : avnd_comp_clc_xxxing_cleansucc_hdler
@@ -1699,31 +1756,11 @@ uint32_t avnd_comp_clc_xxxing_cleansucc_
*/
avnd_comp_cmplete_all_assignment(cb, comp);
- /* If su is restarting then, instantiation of the all the PI components will be done after
- termination of all of them. But for a NPI comp in PI su, restart is done at the time of
- assignment. Such a component never triggers SU FSM. So instantiate it in comp FSM.
- */
- //TODO: Reframe these blockes to make them more illustrative.
- if (!(m_AVND_COMP_TYPE_IS_PREINSTANTIABLE(comp)) && m_AVND_SU_IS_PREINSTANTIABLE(comp->su) &&
- (!m_AVND_SU_IS_RESTART(comp->su)) && (!m_AVND_SU_IS_FAILED(comp->su)))
- /* Instantiate this NPI comp of PI SU if context is not:
- -surestart recovery.
- -restart admin op on SU.*/
- ;//Goahead and instantiate a NPI comp in PI SU.
- else if ((m_AVND_SU_IS_RESTART(comp->su)) && (!m_AVND_SU_IS_FAILED(comp->su)) &&
- (comp->su->admin_op_Id != SA_AMF_ADMIN_RESTART))
- /* Restart the component if it fails with restart reocvery
- during the repair phase of surestart recovery.*/
- ;
- else if (m_AVND_SU_IS_RESTART(comp->su)) {
- if ((comp->pres == SA_AMF_PRESENCE_RESTARTING) && m_AVND_SU_IS_FAILED(comp->su))
- /* Cleanup was already initiated when comp faulted with comprestart recovery.
- If further escalation reached to surestart, same cleanup can be used and thus
- comp can be marked uninstantiated.*/
- avnd_comp_pres_state_set(comp, SA_AMF_PRESENCE_UNINSTANTIATED);
+ //Comp instantiation depends on recovery policy. Check for instantiation eligibility.
+ if (is_failed_comp_eligible_for_instantiation(comp) == false)
goto done;
- }
-
+
+ TRACE("inst_retry_cnt:%u, inst_retry_max:%u",clc_info->inst_retry_cnt,clc_info->inst_retry_max);
if ((clc_info->inst_retry_cnt < clc_info->inst_retry_max) &&
(AVND_COMP_INST_EXIT_CODE_NO_RETRY != clc_info->inst_code_rcvd)) {
/* => keep retrying */
diff --git a/osaf/services/saf/amf/amfnd/include/avnd_su.h b/osaf/services/saf/amf/amfnd/include/avnd_su.h
--- a/osaf/services/saf/amf/amfnd/include/avnd_su.h
+++ b/osaf/services/saf/amf/amfnd/include/avnd_su.h
@@ -421,4 +421,6 @@ bool is_any_non_restartable_comp_assigne
bool su_all_pi_comps_instantiated(const AVND_SU *su);
bool all_csis_in_assigned_state(const AVND_SU *su);
bool isAdminRestarted(const AVND_SU *su);
+bool isFailed(const AVND_SU *su);
+bool isRestartSet(const AVND_SU *su);
#endif
diff --git a/osaf/services/saf/amf/amfnd/su.cc b/osaf/services/saf/amf/amfnd/su.cc
--- a/osaf/services/saf/amf/amfnd/su.cc
+++ b/osaf/services/saf/amf/amfnd/su.cc
@@ -858,3 +858,21 @@ bool isAdminRestarted(const AVND_SU *su)
return (su->admin_op_Id == SA_AMF_ADMIN_RESTART);
}
+/**
+ * @brief Checks if SU is marked failed.
+ * @return true/false
+ */
+bool isFailed(const AVND_SU *su)
+{
+ return (m_AVND_SU_IS_FAILED(su));
+}
+
+/**
+ * @brief Checks if SU is marked restarting because of
+ * su restart recovery or RESTART admin op.
+ * @return true/false
+ */
+bool isRestartSet(const AVND_SU *su)
+{
+ return (m_AVND_SU_IS_RESTART(su));
+}
diff --git a/osaf/services/saf/amf/amfnd/susm.cc b/osaf/services/saf/amf/amfnd/susm.cc
--- a/osaf/services/saf/amf/amfnd/susm.cc
+++ b/osaf/services/saf/amf/amfnd/susm.cc
@@ -1739,6 +1739,11 @@ uint32_t avnd_su_pres_st_chng_prc(AVND_C
/* send the su-oper state msg (to indicate that instantiation failed) */
m_AVND_SU_OPER_STATE_SET(su, SA_AMF_OPERATIONAL_DISABLED);
m_AVND_SEND_CKPT_UPDT_ASYNC_UPDT(cb, su, AVND_CKPT_SU_OPER_STATE);
+ /*No re-assignments and further escalation happens in INST_FAILED state,
+ so reset suRestart flag.*/
+ if (isRestartSet(su))
+ reset_suRestart_flag(su);
+ //Ask AMFD to remove assignments.
rc = avnd_di_oper_send(cb, su, SA_AMF_COMPONENT_FAILOVER);
if (NCSCC_RC_SUCCESS != rc)
goto done;
@@ -1827,6 +1832,11 @@ uint32_t avnd_su_pres_st_chng_prc(AVND_C
m_AVND_SU_OPER_STATE_SET(su, SA_AMF_OPERATIONAL_DISABLED);
m_AVND_SEND_CKPT_UPDT_ASYNC_UPDT(cb, su, AVND_CKPT_SU_OPER_STATE);
+ /*No re-assignments and further escalation happens in INST_FAILED state,
+ so reset suRestart flag.*/
+ if (isRestartSet(su))
+ reset_suRestart_flag(su);
+ //Ask AMFD to remove assignments.
rc = avnd_di_oper_send(cb, su, SA_AMF_COMPONENT_FAILOVER);
}
@@ -2215,7 +2225,7 @@ uint32_t avnd_su_pres_insting_compinstfa
AVND_COMP *curr_comp = 0;
AVND_SU_SI_REC *si = 0;
AVND_COMP_CSI_REC *curr_csi = 0;
- uint32_t rc = NCSCC_RC_SUCCESS;
+ uint32_t rc = NCSCC_RC_SUCCESS, comp_count = 0;
const char *compname = comp ? (char*)comp->name.value : "none";
TRACE_ENTER2("CompInstantiateFailed event in the Instantiate State: '%s' : '%s'",
su->name.value, compname);
@@ -2244,8 +2254,15 @@ uint32_t avnd_su_pres_insting_compinstfa
rc = avnd_comp_clc_fsm_run(cb, curr_comp, AVND_COMP_CLC_PRES_FSM_EV_TERM);
if (NCSCC_RC_SUCCESS != rc)
goto done;
+ comp_count++;
}
} /* for */
+ if (comp_count == 1) {
+ /* If all comps are terminated then set term state and
+ process the SUSI assignment.*/
+ m_AVND_SU_ALL_TERM_SET(su);
+ avnd_su_siq_prc(cb, su);
+ }
}
/*
|
