Menu
▾
▴
[Hyperic-hq-discuss] hq r14402 - in trunk/src/org/hyperic: hq/bizapp/server/action/email hq/events/server/session hq/product/server/mbean util/stats
|
From: <sc...@hy...> - 2010-03-22 05:17:08
|
Author: scottmf Date: 2010-03-21 22:16:57 -0700 (Sun, 21 Mar 2010) New Revision: 14402 URL: http://svn.hyperic.org/?view=rev&root=Hyperic+HQ&revision=14402 Modified: trunk/src/org/hyperic/hq/bizapp/server/action/email/EmailAction.java trunk/src/org/hyperic/hq/events/server/session/AlertConditionsSatisfiedListener.java trunk/src/org/hyperic/hq/product/server/mbean/ProductPluginDeployer.java trunk/src/org/hyperic/util/stats/ConcurrentStatsCollector.java Log: [HHQ-3798] put fireAlert() in a session runner to ensure that there is one session per event to try and avoid the zevent queue from getting backed up. Added stat collection around this area to try to expose the bottlenecks if this should occur again. Modified: trunk/src/org/hyperic/hq/bizapp/server/action/email/EmailAction.java =================================================================== --- trunk/src/org/hyperic/hq/bizapp/server/action/email/EmailAction.java 2010-03-21 07:54:51 UTC (rev 14401) +++ trunk/src/org/hyperic/hq/bizapp/server/action/email/EmailAction.java 2010-03-22 05:16:57 UTC (rev 14402) @@ -95,8 +95,8 @@ // stats on a deployment's activity private static final long EVALUATION_PERIOD = ConcurrentStatsCollector.WRITE_PERIOD*1000; - private static final String EMAIL_ACTIONS = - ConcurrentStatsCollector.EMAIL_ACTIONS; + private static final String SEND_ALERT_TIME = + ConcurrentStatsCollector.SEND_ALERT_TIME; // evaluation window to continue and block notifications or // toggle back to regular operation private static final int THRESHOLD_WINDOW = 10*60*1000; @@ -394,7 +394,7 @@ EmailRecipient[] to, String subject, String[] body, String[] htmlBody, int priority, boolean notifyFiltered) { - ConcurrentStatsCollector.getInstance().addStat(1, EMAIL_ACTIONS); + final long start = System.currentTimeMillis(); if (_alertThreshold <= 0) { final boolean debug = _log.isDebugEnabled(); if (debug) { @@ -411,6 +411,8 @@ _emails.add(obj); } } + ConcurrentStatsCollector.getInstance().addStat( + System.currentTimeMillis()-start, SEND_ALERT_TIME); } private static final EmailManagerLocal getEmailMan() { Modified: trunk/src/org/hyperic/hq/events/server/session/AlertConditionsSatisfiedListener.java =================================================================== --- trunk/src/org/hyperic/hq/events/server/session/AlertConditionsSatisfiedListener.java 2010-03-21 07:54:51 UTC (rev 14401) +++ trunk/src/org/hyperic/hq/events/server/session/AlertConditionsSatisfiedListener.java 2010-03-22 05:16:57 UTC (rev 14402) @@ -7,8 +7,11 @@ import org.apache.commons.logging.LogFactory; import org.hyperic.hibernate.Util; import org.hyperic.hq.events.shared.AlertManagerLocal; +import org.hyperic.hq.hibernate.SessionManager; +import org.hyperic.hq.hibernate.SessionManager.SessionRunner; import org.hyperic.hq.measurement.server.session.AlertConditionsSatisfiedZEvent; import org.hyperic.hq.zevents.ZeventListener; +import org.hyperic.util.stats.ConcurrentStatsCollector; /** * Receives AlertConditionSatisfiedZEvents and forwards them to the AlertManager @@ -22,20 +25,28 @@ private final Log _log = LogFactory.getLog(AlertConditionsSatisfiedListener.class); public void processEvents(List events) { - AlertManagerLocal am = AlertManagerEJBImpl.getOne(); + final AlertManagerLocal am = AlertManagerEJBImpl.getOne(); for (Iterator it=events.iterator(); it.hasNext(); ) { - AlertConditionsSatisfiedZEvent z = (AlertConditionsSatisfiedZEvent)it.next(); + final AlertConditionsSatisfiedZEvent z = (AlertConditionsSatisfiedZEvent)it.next(); + final long start = System.currentTimeMillis(); // HQ-1905 need to retry due to potential StaleStateExceptions for (int ii=0; ii<MAX_RETRIES; ii++) { try { - am.fireAlert(z); + SessionManager.runInSession(new SessionRunner() { + public String getName() { + return "AlertConditionSatifiedListener"; + } + public void run() throws Exception { + am.fireAlert(z); + } + }); break; } catch (Throwable e) { if ((ii+1) < MAX_RETRIES && Util.tranRolledBack(e)) { String times = (MAX_RETRIES - ii == 1) ? "time" : "times"; _log.warn("Warning, exception occurred while running fireAlert. will retry " - + (MAX_RETRIES - (ii+1)) + " more " + times + ". errorMsg: " + e); + + (MAX_RETRIES - ii) + " more " + times + ". errorMsg: " + e); continue; } else { _log.error("fireAlert threw an Exception, will not be retried",e); @@ -43,6 +54,8 @@ } } } + ConcurrentStatsCollector.getInstance().addStat( + System.currentTimeMillis()-start, ConcurrentStatsCollector.FIRED_ALERT_TIME); } } Modified: trunk/src/org/hyperic/hq/product/server/mbean/ProductPluginDeployer.java =================================================================== --- trunk/src/org/hyperic/hq/product/server/mbean/ProductPluginDeployer.java 2010-03-21 07:54:51 UTC (rev 14401) +++ trunk/src/org/hyperic/hq/product/server/mbean/ProductPluginDeployer.java 2010-03-22 05:16:57 UTC (rev 14402) @@ -276,11 +276,12 @@ c.register(ConcurrentStatsCollector.PURGE_EVENT_LOGS_TIME); c.register(ConcurrentStatsCollector.PURGE_MEASUREMENTS_TIME); c.register(ConcurrentStatsCollector.MEASUREMENT_SCHEDULE_TIME); - c.register(ConcurrentStatsCollector.EMAIL_ACTIONS); + c.register(ConcurrentStatsCollector.SEND_ALERT_TIME); c.register(ConcurrentStatsCollector.ZEVENT_QUEUE_SIZE); c.register(ConcurrentStatsCollector.FIRE_ALERT_TIME); c.register(ConcurrentStatsCollector.EVENT_PROCESSING_TIME); c.register(ConcurrentStatsCollector.TRIGGER_INIT_TIME); + c.register(ConcurrentStatsCollector.FIRED_ALERT_TIME); c.startCollector(); } catch (Exception e) { _log.error("Could not start Concurrent Stats Collector", e); Modified: trunk/src/org/hyperic/util/stats/ConcurrentStatsCollector.java =================================================================== --- trunk/src/org/hyperic/util/stats/ConcurrentStatsCollector.java 2010-03-21 07:54:51 UTC (rev 14401) +++ trunk/src/org/hyperic/util/stats/ConcurrentStatsCollector.java 2010-03-22 05:16:57 UTC (rev 14402) @@ -106,9 +106,10 @@ PURGE_EVENT_LOGS_TIME = "PURGE_EVENT_LOGS_TIME", PURGE_MEASUREMENTS_TIME = "PURGE_MEASUREMENTS_TIME", MEASUREMENT_SCHEDULE_TIME = "MEASUREMENT_SCHEDULE_TIME", - EMAIL_ACTIONS = "EMAIL_ACTIONS", + SEND_ALERT_TIME = "SEND_ALERT_TIME", ZEVENT_QUEUE_SIZE = "ZEVENT_QUEUE_SIZE", - TRIGGER_INIT_TIME = "TRIGGER_INIT_TIME"; + TRIGGER_INIT_TIME = "TRIGGER_INIT_TIME", + FIRED_ALERT_TIME = "FIRED_ALERT_TIME"; // using tree due to ordering capabilities private final Map _statKeys = new TreeMap(); private AtomicBoolean _hasStarted = new AtomicBoolean(false); |
