From: <tk...@hy...> - 2007-11-12 17:37:30
|
Author: tkeeney Date: 2007-11-12 09:37:26 -0800 (Mon, 12 Nov 2007) New Revision: 6749 URL: http://svn.hyperic.org/?view=rev&root=Hyperic+HQ&revision=6749 Added: trunk/src/org/hyperic/hq/events/AlertDefinitionLastFiredUpdateEvent.java trunk/src/org/hyperic/hq/events/AlertDefinitionLastFiredUpdateEvent_test.java trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionLastFiredCallback.java trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionLastFiredTimeUpdater.java Modified: trunk/src/org/hyperic/hq/bizapp/server/mdb/RegisteredDispatcherEJBImpl.java trunk/src/org/hyperic/hq/events/AbstractEvent.java trunk/src/org/hyperic/hq/events/ext/AbstractTrigger.java trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionDAO.java trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionManagerEJBImpl.java trunk/src/org/hyperic/hq/events/server/session/AlertManagerEJBImpl.java trunk/src/org/hyperic/hq/events/server/session/ClassicEscalatableCreator.java trunk/src/org/hyperic/hq/events/server/session/EventsStartupListener.java Log: [HHQ-1195] To reduce the possibility of deadlocking within the alert definition table when 2 concurrent threads update the last fired time on the same alert definition, we are now serializing updates to the last fired time in a separate executor thread (the alert definition last fired time updater) after the registered dispatcher commits its txn. Modified: trunk/src/org/hyperic/hq/bizapp/server/mdb/RegisteredDispatcherEJBImpl.java =================================================================== --- trunk/src/org/hyperic/hq/bizapp/server/mdb/RegisteredDispatcherEJBImpl.java 2007-11-12 17:32:34 UTC (rev 6748) +++ trunk/src/org/hyperic/hq/bizapp/server/mdb/RegisteredDispatcherEJBImpl.java 2007-11-12 17:37:26 UTC (rev 6749) @@ -25,10 +25,11 @@ package org.hyperic.hq.bizapp.server.mdb; -import java.util.ArrayList; +import java.io.Serializable; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Set; @@ -41,6 +42,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.hyperic.hq.application.HQApp; +import org.hyperic.hq.application.TransactionListener; import org.hyperic.hq.bizapp.server.trigger.conditional.MultiConditionTrigger; import org.hyperic.hq.common.util.Messenger; import org.hyperic.hq.events.AbstractEvent; @@ -50,6 +53,7 @@ import org.hyperic.hq.events.FlushStateEvent; import org.hyperic.hq.events.TriggerInterface; import org.hyperic.hq.events.ext.RegisteredTriggers; +import org.hyperic.hq.events.server.session.AlertDefinitionLastFiredTimeUpdater; /** The RegisteredDispatcher Message-Drive Bean registers Triggers and @@ -212,14 +216,61 @@ List enqueuedEvents = Messenger.drainEnqueuedMessages(); if (!enqueuedEvents.isEmpty()) { - final ArrayList eventsToPublish = new ArrayList(enqueuedEvents); - + LinkedList eventsToPublish = new LinkedList(); + LinkedList alertDefLastFiredEventsToPublish = new LinkedList(); + + for (Iterator iter = enqueuedEvents.iterator(); iter.hasNext();) { + AbstractEvent event = (AbstractEvent) iter.next(); + + if (event.isAlertDefinitionLastFiredUpdateEvent()) { + alertDefLastFiredEventsToPublish.add(event); + } else { + eventsToPublish.add(event); + } + } + + publishOnEventsTopicNow(eventsToPublish); + + // To reduce contention on the alert def table, publish alert def + // last fired time updates post commit. + publishAlertDefLastFiredEventsPostCommit(alertDefLastFiredEventsToPublish); + + } + } + + private void publishOnEventsTopicNow(List events) { + if (!events.isEmpty()) { Messenger sender = new Messenger(); - sender.publishMessage(EventConstants.EVENTS_TOPIC, - eventsToPublish); + sender.publishMessage(EventConstants.EVENTS_TOPIC, (Serializable)events); } } + + private void publishAlertDefLastFiredEventsPostCommit(final List events) { + if (!events.isEmpty()) { + try { + HQApp.getInstance().addTransactionListener(new TransactionListener() { + public void afterCommit(boolean success) { + try { + AlertDefinitionLastFiredTimeUpdater.getInstance() + .enqueueEvents(events); + } catch (InterruptedException e) { + // we've been interrupted - oh well + } + } + public void beforeCommit() { + } + }); + } catch (Throwable t) { + // We want to complete the current txn even if registering + // the post commit listener fails. + log.warn("Failed to publish the alert definition last fired " + + "events. The last fired time may not be updated immediately " + + "for some alert definitions: "+events, t); + } + } + } + /** * @ejb:create-method */ Modified: trunk/src/org/hyperic/hq/events/AbstractEvent.java =================================================================== --- trunk/src/org/hyperic/hq/events/AbstractEvent.java 2007-11-12 17:32:34 UTC (rev 6748) +++ trunk/src/org/hyperic/hq/events/AbstractEvent.java 2007-11-12 17:37:26 UTC (rev 6749) @@ -72,4 +72,8 @@ public boolean isLoggingSupported() { return this instanceof LoggableInterface; } + + public boolean isAlertDefinitionLastFiredUpdateEvent() { + return this instanceof AlertDefinitionLastFiredUpdateEvent; + } } Added: trunk/src/org/hyperic/hq/events/AlertDefinitionLastFiredUpdateEvent.java =================================================================== --- trunk/src/org/hyperic/hq/events/AlertDefinitionLastFiredUpdateEvent.java (rev 0) +++ trunk/src/org/hyperic/hq/events/AlertDefinitionLastFiredUpdateEvent.java 2007-11-12 17:37:26 UTC (rev 6749) @@ -0,0 +1,122 @@ +/* + * NOTE: This copyright does *not* cover user programs that use HQ + * program services by normal system calls through the application + * program interfaces provided as part of the Hyperic Plug-in Development + * Kit or the Hyperic Client Development Kit - this is merely considered + * normal use of the program, and does *not* fall under the heading of + * "derived work". + * + * Copyright (C) [2004, 2005, 2006], Hyperic, Inc. + * This file is part of HQ. + * + * HQ is free software; you can redistribute it and/or modify + * it under the terms version 2 of the GNU General Public License as + * published by the Free Software Foundation. This program is distributed + * in the hope that it will be useful, but WITHOUT ANY WARRANTY; without + * even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + */ + +package org.hyperic.hq.events; + +import java.io.Serializable; + +import org.hyperic.hq.events.server.session.AlertDefinition; + +/** + * An event indicating that an alert definition last fired time has + * been updated to a new value. + */ +public class AlertDefinitionLastFiredUpdateEvent extends AbstractEvent + implements Serializable, Comparable { + + private static final long serialVersionUID = -8148649541156405020L; + + /** + * Creates an instance. + * + * @param alertDef The alert definition. + * @param lastFiredTime The new last fired time. + */ + public AlertDefinitionLastFiredUpdateEvent(AlertDefinition alertDef, + long lastFiredTime) { + this(alertDef.getId(), lastFiredTime); + } + + /** + * Creates an instance. + * + * @param alertDefId The alert definition id. + * @param lastFiredTime The new last fired time. + */ + public AlertDefinitionLastFiredUpdateEvent(Integer alertDefId, + long lastFiredTime) { + this.setInstanceId(alertDefId); + this.setTimestamp(lastFiredTime); + } + + /** + * Retrieve the alert definition id. + * + * @return The alert definition id. + */ + public Integer getAlertDefinitionId() { + return getInstanceId(); + } + + /** + * Retrieve the last fired time. + * + * @return The last fired time in milliseconds. + */ + public long getLastFiredTime() { + return getTimestamp(); + } + + public int compareTo(Object o) { + AlertDefinitionLastFiredUpdateEvent event + = (AlertDefinitionLastFiredUpdateEvent)o; + + if (this.getLastFiredTime() != event.getLastFiredTime()) { + return (int)(this.getLastFiredTime() - + event.getLastFiredTime()); + } else { + return this.getAlertDefinitionId().intValue() - + event.getAlertDefinitionId().intValue(); + } + } + + public boolean equals(Object o) { + if (o == this) { + return true; + } + + if (o instanceof AlertDefinitionLastFiredUpdateEvent) { + AlertDefinitionLastFiredUpdateEvent event + = (AlertDefinitionLastFiredUpdateEvent)o; + + return this.getLastFiredTime() == event.getLastFiredTime() && + this.getAlertDefinitionId().equals(event.getAlertDefinitionId()); + } + + return false; + } + + public int hashCode() { + int result = 37; + result = 17*result + (int)this.getLastFiredTime(); + result = 17*result + this.getAlertDefinitionId().hashCode(); + return result; + } + + public String toString() { + return getAlertDefinitionId()+":"+getLastFiredTime(); + } + +} Added: trunk/src/org/hyperic/hq/events/AlertDefinitionLastFiredUpdateEvent_test.java =================================================================== --- trunk/src/org/hyperic/hq/events/AlertDefinitionLastFiredUpdateEvent_test.java (rev 0) +++ trunk/src/org/hyperic/hq/events/AlertDefinitionLastFiredUpdateEvent_test.java 2007-11-12 17:37:26 UTC (rev 6749) @@ -0,0 +1,160 @@ +/* + * NOTE: This copyright does *not* cover user programs that use HQ + * program services by normal system calls through the application + * program interfaces provided as part of the Hyperic Plug-in Development + * Kit or the Hyperic Client Development Kit - this is merely considered + * normal use of the program, and does *not* fall under the heading of + * "derived work". + * + * Copyright (C) [2004, 2005, 2006], Hyperic, Inc. + * This file is part of HQ. + * + * HQ is free software; you can redistribute it and/or modify + * it under the terms version 2 of the GNU General Public License as + * published by the Free Software Foundation. This program is distributed + * in the hope that it will be useful, but WITHOUT ANY WARRANTY; without + * even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + */ + +package org.hyperic.hq.events; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +import junit.framework.TestCase; + +/** + * Tests the AlertDefinitionLastFiredUpdateEvent class. + */ +public class AlertDefinitionLastFiredUpdateEvent_test extends TestCase { + + public AlertDefinitionLastFiredUpdateEvent_test(String name) { + super(name); + } + + /** + * Test compareTo() and that comparable objects are equal. + */ + public void testCompareToAndEquals() { + Integer id1 = new Integer(1); + Integer id2 = new Integer(2); + + long time1 = 1000; + long time2 = 2000; + + AlertDefinitionLastFiredUpdateEvent event1 = + new AlertDefinitionLastFiredUpdateEvent(id1, time1); + + AlertDefinitionLastFiredUpdateEvent event2 = + new AlertDefinitionLastFiredUpdateEvent(id1, time1); + + AlertDefinitionLastFiredUpdateEvent event3 = + new AlertDefinitionLastFiredUpdateEvent(id1, time2); + + AlertDefinitionLastFiredUpdateEvent event4 = + new AlertDefinitionLastFiredUpdateEvent(id2, time1); + + assertEquals(0, event1.compareTo(event1)); + assertTrue(event1.equals(event1)); + + assertEquals(0, event1.compareTo(event2)); + assertTrue(event1.equals(event2)); + + assertTrue(event1.compareTo(event3) < 0); + assertTrue(event3.compareTo(event1) > 0); + assertFalse(event1.equals(event3)); + + assertTrue(event1.compareTo(event4) < 0); + assertTrue(event4.compareTo(event1) > 0); + assertFalse(event1.equals(event4)); + + // note that we compare last fired times first, then alert def ids. + assertTrue(event3.compareTo(event4) > 0); + assertTrue(event4.compareTo(event3) < 0); + assertFalse(event3.equals(event4)); + } + + /** + * Test that equal objects have equal hash codes. + */ + public void testHashCode() { + Integer id1 = new Integer(1); + + long time1 = 1000; + + AlertDefinitionLastFiredUpdateEvent event1 = + new AlertDefinitionLastFiredUpdateEvent(id1, time1); + + AlertDefinitionLastFiredUpdateEvent event2 = + new AlertDefinitionLastFiredUpdateEvent(id1, time1); + + assertTrue(event1.equals(event1)); + assertEquals(event1.hashCode(), event1.hashCode()); + + assertTrue(event1.equals(event2)); + assertEquals(event1.hashCode(), event2.hashCode()); + } + + /** + * Test sorting the alert def events, largest to smallest. We sort first + * by last fired time, then by alert def id. This is how we sort these + * events in the alert def last fired time updater. + */ + public void testSortingAlertDefEvents() { + Integer id1 = new Integer(1); + Integer id2 = new Integer(2); + + long time1 = 1000; + long time2 = 2000; + + AlertDefinitionLastFiredUpdateEvent event1 = + new AlertDefinitionLastFiredUpdateEvent(id1, time1); + + AlertDefinitionLastFiredUpdateEvent event2 = + new AlertDefinitionLastFiredUpdateEvent(id1, time1); + + AlertDefinitionLastFiredUpdateEvent event3 = + new AlertDefinitionLastFiredUpdateEvent(id1, time2); + + AlertDefinitionLastFiredUpdateEvent event4 = + new AlertDefinitionLastFiredUpdateEvent(id2, time1); + + AlertDefinitionLastFiredUpdateEvent event5 = + new AlertDefinitionLastFiredUpdateEvent(id2, time2); + + // sorting largest to smallest - use a reverse order comparator + Set orderedEvents = new TreeSet(Collections.reverseOrder()); + + orderedEvents.add(event1); + orderedEvents.add(event2); + orderedEvents.add(event3); + orderedEvents.add(event4); + orderedEvents.add(event5); + + // this is a set so there should be only one instance of equal elements + // in the collection + orderedEvents.add(event1); + orderedEvents.add(event4); + + List doOrdering = new ArrayList(orderedEvents); + + List expectedOrder = new ArrayList(); + expectedOrder.add(event5); + expectedOrder.add(event3); + expectedOrder.add(event4); + expectedOrder.add(event1); + + assertEquals(expectedOrder, doOrdering); + } + +} Modified: trunk/src/org/hyperic/hq/events/ext/AbstractTrigger.java =================================================================== --- trunk/src/org/hyperic/hq/events/ext/AbstractTrigger.java 2007-11-12 17:32:34 UTC (rev 6748) +++ trunk/src/org/hyperic/hq/events/ext/AbstractTrigger.java 2007-11-12 17:37:26 UTC (rev 6749) @@ -50,11 +50,13 @@ import org.hyperic.hq.events.AbstractEvent; import org.hyperic.hq.events.ActionExecuteException; import org.hyperic.hq.events.AlertCreateException; +import org.hyperic.hq.events.AlertDefinitionLastFiredUpdateEvent; import org.hyperic.hq.events.EventConstants; import org.hyperic.hq.events.TriggerFiredEvent; import org.hyperic.hq.events.TriggerInterface; import org.hyperic.hq.events.TriggerNotFiredEvent; import org.hyperic.hq.events.server.session.AlertDefinition; +import org.hyperic.hq.events.server.session.AlertDefinitionLastFiredCallback; import org.hyperic.hq.events.server.session.AlertDefinitionManagerEJBImpl; import org.hyperic.hq.events.server.session.AlertManagerEJBImpl; import org.hyperic.hq.events.server.session.ClassicEscalatableCreator; @@ -85,6 +87,7 @@ private static ObjectName readyManName; private RegisteredTriggerValue triggerValue = new RegisteredTriggerValue(); + public AbstractTrigger() { super(); @@ -182,7 +185,9 @@ } EscalatableCreator creator = - new ClassicEscalatableCreator(alertDef, event); + new ClassicEscalatableCreator(alertDef, + event, + getAlertDefLastFiredCallback()); // Now start escalation if (alertDef.getEscalation() != null) { @@ -200,7 +205,7 @@ "Overlord does not have permission to disable definition"); } } - + private boolean shouldFireActions(AlertDefinitionManagerLocal aman, AlertDefinition alertDef) throws PermissionException { @@ -268,6 +273,17 @@ } + private AlertDefinitionLastFiredCallback getAlertDefLastFiredCallback() { + return new AlertDefinitionLastFiredCallback() { + public void onLastFiredUpdate(AlertDefinition alertDef, + long lastFiredTime) { + AbstractEvent event = + new AlertDefinitionLastFiredUpdateEvent(alertDef, lastFiredTime); + AbstractTrigger.this.publishEvent(event); + } + }; + } + /** * Deserialize an event from the input stream, providing optional recovery * from stream corruption. The stream may become corrupted during upgrade Modified: trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionDAO.java =================================================================== --- trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionDAO.java 2007-11-12 17:32:34 UTC (rev 6748) +++ trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionDAO.java 2007-11-12 17:37:26 UTC (rev 6749) @@ -24,6 +24,7 @@ */ package org.hyperic.hq.events.server.session; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -38,6 +39,7 @@ import org.hyperic.hq.authz.shared.AuthzSubjectValue; import org.hyperic.hq.authz.shared.PermissionManagerFactory; import org.hyperic.hq.dao.HibernateDAO; +import org.hyperic.hq.events.AlertDefinitionLastFiredUpdateEvent; import org.hyperic.hq.events.AlertSeverity; import org.hyperic.hq.events.EventConstants; import org.hyperic.hq.events.shared.ActionValue; @@ -353,4 +355,54 @@ return pInfo.pageResults(q).list(); } + + /** + * Update in batch the alert definitions last fired times for each of the + * provided events, using batch size specified by the + * <code>hibernate.jdbc.batch_size</code> configuration property. + * + * @param events The update events. + */ + void updateAlertDefinitionsLastFiredTimes(AlertDefinitionLastFiredUpdateEvent[] events) { + String sql = "update AlertDefinition ad set ad.lastFired = :lastFiredTime " + + "where ad.id = :defid and (ad.lastFired is null or ad.lastFired < :lastFiredTime)"; + + Session session = getSession(); + Query query = session.createQuery(sql); + + for (int i = 0; i < events.length; i++) { + AlertDefinitionLastFiredUpdateEvent event = events[i]; + query.setInteger("defid", event.getAlertDefinitionId().intValue()); + query.setLong("lastFiredTime", event.getLastFiredTime()); + query.executeUpdate(); + } + + session.flush(); + } + + /** + * Find all alerts with ctimes greater than the last fired time in the + * associated alert definition. Return the alert ctimes as alert definition + * last fired time update events. This method is used to confirm that alert + * definitions have last fired times at least equal to the ctimes specified + * for their associated alerts. + * + * @return The list of {@link AlertDefinitionLastFiredUpdateEvent update events}. + */ + List getEventsForAlertDefinitionsWithOldLastFiredTimes() { + String sql = "select ad.id, a.ctime from Alert a join a.alertDefinition ad " + + "where ad.lastFired is null or ad.lastFired < a.ctime"; + + List results = getSession().createQuery(sql).list(); + List events = new ArrayList(results.size()); + + for (Iterator iter = results.iterator(); iter.hasNext();) { + Object[] result = (Object[]) iter.next(); + events.add(new AlertDefinitionLastFiredUpdateEvent((Integer)result[0], + ((Long)result[1]).longValue())); + } + + return events; + } + } Added: trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionLastFiredCallback.java =================================================================== --- trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionLastFiredCallback.java (rev 0) +++ trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionLastFiredCallback.java 2007-11-12 17:37:26 UTC (rev 6749) @@ -0,0 +1,42 @@ +/* + * NOTE: This copyright does *not* cover user programs that use HQ + * program services by normal system calls through the application + * program interfaces provided as part of the Hyperic Plug-in Development + * Kit or the Hyperic Client Development Kit - this is merely considered + * normal use of the program, and does *not* fall under the heading of + * "derived work". + * + * Copyright (C) [2004, 2005, 2006], Hyperic, Inc. + * This file is part of HQ. + * + * HQ is free software; you can redistribute it and/or modify + * it under the terms version 2 of the GNU General Public License as + * published by the Free Software Foundation. This program is distributed + * in the hope that it will be useful, but WITHOUT ANY WARRANTY; without + * even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + */ + +package org.hyperic.hq.events.server.session; + +/** + * A callback for notifying an interested party if the alert definition last + * fired time has been updated. + */ +public interface AlertDefinitionLastFiredCallback { + + /** + * Called on update to the alert definition last fired time. + * + * @param alertDef The alert definition. + * @param lastFiredTime The new last fired time. + */ + void onLastFiredUpdate(AlertDefinition alertDef, long lastFiredTime); + +} Added: trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionLastFiredTimeUpdater.java =================================================================== --- trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionLastFiredTimeUpdater.java (rev 0) +++ trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionLastFiredTimeUpdater.java 2007-11-12 17:37:26 UTC (rev 6749) @@ -0,0 +1,224 @@ +/* + * NOTE: This copyright does *not* cover user programs that use HQ + * program services by normal system calls through the application + * program interfaces provided as part of the Hyperic Plug-in Development + * Kit or the Hyperic Client Development Kit - this is merely considered + * normal use of the program, and does *not* fall under the heading of + * "derived work". + * + * Copyright (C) [2004, 2005, 2006], Hyperic, Inc. + * This file is part of HQ. + * + * HQ is free software; you can redistribute it and/or modify + * it under the terms version 2 of the GNU General Public License as + * published by the Free Software Foundation. This program is distributed + * in the hope that it will be useful, but WITHOUT ANY WARRANTY; without + * even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + */ + +package org.hyperic.hq.events.server.session; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.hyperic.hq.application.ShutdownCallback; +import org.hyperic.hq.events.AlertDefinitionLastFiredUpdateEvent; +import org.hyperic.hq.events.shared.AlertDefinitionManagerLocal; + +import EDU.oswego.cs.dl.util.concurrent.LinkedQueue; +import EDU.oswego.cs.dl.util.concurrent.QueuedExecutor; +import EDU.oswego.cs.dl.util.concurrent.ThreadFactory; + +/** + * A singleton that queues and processes + * {@link AlertDefinitionLastFiredUpdateEvent AlertDefinitionLastFiredUpdateEvents} + * by updating each alert definition last fired time associated with each + * enqueued event. + */ +public class AlertDefinitionLastFiredTimeUpdater implements ShutdownCallback { + + private static final AlertDefinitionLastFiredTimeUpdater UPDATER = + new AlertDefinitionLastFiredTimeUpdater(); + + // The LRU cache size - we don't need an aggressive cache so 100 is + // an ok default value + private static final int MAX_ENTRIES = + Integer.getInteger("org.hq.alertdef.updater.max.cache.size", 100).intValue(); + + private static final Log log = + LogFactory.getLog(AlertDefinitionLastFiredTimeUpdater.class); + + private final QueuedExecutor executor; + + private final Map alertDefId2LastFiredTimeCache; + + + private AlertDefinitionLastFiredTimeUpdater() { + executor = new QueuedExecutor(new LinkedQueue()); + + executor.setThreadFactory(new ThreadFactory() { + public Thread newThread(Runnable runnable) { + Thread thread = new Thread(runnable, "AlertDef LastFired Updater"); + thread.setDaemon(true); + return thread; + }}); + + // warm up the executor + executor.restart(); + + alertDefId2LastFiredTimeCache = new LinkedHashMap(16, 0.75f, true) { + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > MAX_ENTRIES; + } + }; + } + + /** + * @return The instance of this singleton. + */ + public static AlertDefinitionLastFiredTimeUpdater getInstance() { + return UPDATER; + } + + /** + * Enqueue the events for processing. We are making the optimizing assumption + * that all the events in the collection are of type + * {@link AlertDefinitionLastFiredUpdateEvent AlertDefinitionLastFiredUpdateEvent}. + * It is up to the client to enforce this precondition. + * + * @param events The events to process. + * @throws InterruptedException if event enqueuing has been interrupted. + */ + public void enqueueEvents(final Collection events) throws InterruptedException { + executor.execute(new Updater(events, alertDefId2LastFiredTimeCache)); + } + + /** + * The shutdown hook invokes this callback. + */ + public void shutdown() { + // Force the shutdown since we can't be guaranteed that the alert def + // session bean will be operational on shutdown. + shutdown(true); + } + + /** + * Shutdown the updater. + * + * @param force <code>true</code> to force shutdown immediately; + * <code>false</code> to process the events currently enqueued. + */ + public void shutdown(boolean force) { + if (force) { + executor.shutdownNow(); + } else { + executor.shutdownAfterProcessingCurrentlyQueuedTasks(); + } + } + + /** + * The class that does all the work! + */ + private static class Updater implements Runnable { + + private final Collection _events; + private final Map _alertDefId2LastFiredTimeCache; + + public Updater(Collection events, Map alertDefId2LastFiredTimeCache) { + _events = events; + _alertDefId2LastFiredTimeCache = alertDefId2LastFiredTimeCache; + } + + public void run() { + // throw out any events with old alert def last fired times + AlertDefinitionLastFiredUpdateEvent[] updateEvents = getRelevantEvents(); + + tryUpdateAlertDefsLastFiredTimes(updateEvents); + } + + private AlertDefinitionLastFiredUpdateEvent[] getRelevantEvents() { + Set toUpdate = new HashSet(); + + final Map cache = _alertDefId2LastFiredTimeCache; + + // order the events, latest timestamp first + Set orderedEvents = new TreeSet(Collections.reverseOrder()); + orderedEvents.addAll(_events); + _events.clear(); + + for (Iterator iter = orderedEvents.iterator(); iter.hasNext();) { + AlertDefinitionLastFiredUpdateEvent event = + (AlertDefinitionLastFiredUpdateEvent) iter.next(); + + Long currLastFiredTime = (Long)cache.get(event.getAlertDefinitionId()); + + if (currLastFiredTime == null || + currLastFiredTime.longValue() < event.getLastFiredTime()) { + + cache.put(event.getAlertDefinitionId(), + new Long(event.getLastFiredTime())); + + toUpdate.add(event); + } + } + + return (AlertDefinitionLastFiredUpdateEvent[]) + toUpdate.toArray( + new AlertDefinitionLastFiredUpdateEvent[toUpdate.size()]); + } + + /** + * Update the alert def last fired times within an EJB call to start a + * transactional context. Use exponential backoff for failed updates. + * + * @param updateEvents The update events. + */ + private void tryUpdateAlertDefsLastFiredTimes( + AlertDefinitionLastFiredUpdateEvent[] updateEvents) { + long sleep = 10; + int numTries = 0; + boolean succeeded = false; + + while (succeeded==false) { + try { + AlertDefinitionManagerLocal alertDefMan = + AlertDefinitionManagerEJBImpl.getOne(); + + alertDefMan.updateAlertDefinitionsLastFiredTimes(updateEvents); + succeeded = true; + } catch (Throwable e) { + if (++numTries == 5) { + log.error("Failed to update alert definition last " + + "fired times: "+updateEvents, e); + break; + } + + try { + Thread.sleep(sleep); + } catch (InterruptedException e1) { + // ignore + } + + sleep = (sleep*3/2)+1; + } + } + } + + } + +} Modified: trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionManagerEJBImpl.java =================================================================== --- trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionManagerEJBImpl.java 2007-11-12 17:32:34 UTC (rev 6748) +++ trunk/src/org/hyperic/hq/events/server/session/AlertDefinitionManagerEJBImpl.java 2007-11-12 17:37:26 UTC (rev 6749) @@ -26,6 +26,7 @@ package org.hyperic.hq.events.server.session; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -56,6 +57,7 @@ import org.hyperic.hq.events.ActionCreateException; import org.hyperic.hq.events.AlertConditionCreateException; import org.hyperic.hq.events.AlertDefinitionCreateException; +import org.hyperic.hq.events.AlertDefinitionLastFiredUpdateEvent; import org.hyperic.hq.events.AlertSeverity; import org.hyperic.hq.events.EventConstants; import org.hyperic.hq.events.shared.ActionValue; @@ -270,7 +272,40 @@ return res.getAlertDefinitionValue(); } + /** + * Update alert definitions last fired times for each of the provided + * update events. + * + * @param events The update events. + * @ejb:interface-method + */ + public void updateAlertDefinitionsLastFiredTimes(AlertDefinitionLastFiredUpdateEvent[] events) { + + getAlertDefDAO().updateAlertDefinitionsLastFiredTimes(events); + + } + /** + * Synchronize the alert definitions last fired times with their associated + * alerts. If an alert definition exists with a null last fired time or + * a last fired time that is less than the greatest ctime for the associated + * alerts, then set that alert definition's last fired time to the alert + * ctime. + * + * @ejb:interface-method + */ + public void synchAlertDefinitionsLastFiredTimes() { + List events = getAlertDefDAO() + .getEventsForAlertDefinitionsWithOldLastFiredTimes(); + + try { + AlertDefinitionLastFiredTimeUpdater.getInstance().enqueueEvents(events); + } catch (InterruptedException e) { + // do nothing + } + } + + /** * Update just the basics * @throws PermissionException * Modified: trunk/src/org/hyperic/hq/events/server/session/AlertManagerEJBImpl.java =================================================================== --- trunk/src/org/hyperic/hq/events/server/session/AlertManagerEJBImpl.java 2007-11-12 17:32:34 UTC (rev 6748) +++ trunk/src/org/hyperic/hq/events/server/session/AlertManagerEJBImpl.java 2007-11-12 17:37:26 UTC (rev 6749) @@ -67,6 +67,7 @@ import org.hyperic.hq.measurement.server.session.DerivedMeasurement; import org.hyperic.hq.measurement.server.session.DerivedMeasurementDAO; import org.hyperic.hq.measurement.shared.ResourceLogEvent; +import org.hyperic.hq.events.server.session.AlertDefinitionLastFiredCallback; import org.hyperic.util.NumberUtil; import org.hyperic.util.pager.PageControl; import org.hyperic.util.pager.PageList; @@ -124,8 +125,11 @@ } /** - * Create a new alert - * + * Create a new alert, setting the alert definition last fired time + * immediately. + * + * @param def The alert definition. + * @param ctime The alert creation time. * @ejb:interface-method */ public Alert createAlert(AlertDefinition def, long ctime) { @@ -136,6 +140,27 @@ getAlertDAO().save(alert); return alert; } + + /** + * Create a new alert and notify the alert definition last fired time + * callback that the alert definition last fired time should be updated. + * Do not actually update the last fired time. + * + * @param def The alert definition. + * @param ctime The alert creation time. + * @param callback The alert definition last fired time callback. + * @ejb:interface-method + */ + public Alert createAlert(AlertDefinition def, + long ctime, + AlertDefinitionLastFiredCallback callback) { + Alert alert = new Alert(); + alert.setAlertDefinition(def); + alert.setCtime(ctime); + callback.onLastFiredUpdate(def, ctime); + getAlertDAO().save(alert); + return alert; + } /** * Simply mark an alert object as fixed Modified: trunk/src/org/hyperic/hq/events/server/session/ClassicEscalatableCreator.java =================================================================== --- trunk/src/org/hyperic/hq/events/server/session/ClassicEscalatableCreator.java 2007-11-12 17:32:34 UTC (rev 6748) +++ trunk/src/org/hyperic/hq/events/server/session/ClassicEscalatableCreator.java 2007-11-12 17:37:26 UTC (rev 6749) @@ -54,16 +54,42 @@ private static final Log _log = LogFactory.getLog(ClassicEscalatableCreator.class); - private AlertDefinition _def; - private TriggerFiredEvent _event; + private final AlertDefinition _def; + private final TriggerFiredEvent _event; + private final AlertDefinitionLastFiredCallback _callback; + /** + * Creates an instance that sets the alert definition last fired time + * immediately when the alert is created. + * + * @param def The alert definition. + * @param event The event that triggered the escalation. + */ public ClassicEscalatableCreator(AlertDefinition def, TriggerFiredEvent event) { _def = def; _event = event; + _callback = null; } /** + * Creates an instance that registers a callback to be notified when a new + * value should be set for the alert definition last fired time. This value + * is not actually set on the alert definition. + * + * @param def The alert definition. + * @param event The event that triggered the escalation. + * @param callback The callback. + */ + public ClassicEscalatableCreator(AlertDefinition def, + TriggerFiredEvent event, + AlertDefinitionLastFiredCallback callback) { + _def = def; + _event = event; + _callback = callback; + } + + /** * In the classic escalatable architecture, we still need to support the * execution of the actions defined for the regular alert defintion * (in addition to executing the actions specified by the escalation). @@ -82,7 +108,13 @@ } // Now create the alert - Alert alert = alertMan.createAlert(_def, _event.getTimestamp()); + Alert alert = null; + + if (_callback == null) { + alert = alertMan.createAlert(_def, _event.getTimestamp()); + } else { + alert = alertMan.createAlert(_def, _event.getTimestamp(), _callback); + } // Create a alert condition logs for every condition that triggered the alert Collection conds = _def.getConditions(); Modified: trunk/src/org/hyperic/hq/events/server/session/EventsStartupListener.java =================================================================== --- trunk/src/org/hyperic/hq/events/server/session/EventsStartupListener.java 2007-11-12 17:32:34 UTC (rev 6748) +++ trunk/src/org/hyperic/hq/events/server/session/EventsStartupListener.java 2007-11-12 17:37:26 UTC (rev 6749) @@ -25,6 +25,7 @@ package org.hyperic.hq.events.server.session; +import java.util.Arrays; import java.util.StringTokenizer; import javax.management.MBeanServer; @@ -35,7 +36,9 @@ import org.apache.commons.logging.LogFactory; import org.hibernate.PropertyNotFoundException; import org.hyperic.hq.application.HQApp; +import org.hyperic.hq.application.ShutdownCallback; import org.hyperic.hq.application.StartupListener; +import org.hyperic.hq.events.AlertDefinitionLastFiredUpdateEvent; public class EventsStartupListener implements StartupListener @@ -52,6 +55,11 @@ AlertableRoleCalendarType.class.getClass(); HQApp app = HQApp.getInstance(); + + // Register the alert def last fired time updater with the + // shutdown hook. + app.registerCallbackListener(ShutdownCallback.class, + AlertDefinitionLastFiredTimeUpdater.getInstance()); synchronized (LOCK) { _changeCallback = (TriggerChangeCallback) @@ -60,8 +68,10 @@ loadConfigProps("triggers"); loadConfigProps("actions"); + + synchAlertDefinitionsLastFiredTimes(); } - + private void loadConfigProps(String prop) { try { MBeanServer mServer = (MBeanServer) MBeanServerFactory @@ -105,6 +115,16 @@ } } + /** + * We need to make sure that the alert definition last fired times are + * up to date. + */ + private void synchAlertDefinitionsLastFiredTimes() { + _log.debug("Synching the alert definition last fired times."); + + AlertDefinitionManagerEJBImpl.getOne().synchAlertDefinitionsLastFiredTimes(); + } + static TriggerChangeCallback getChangedTriggerCallback() { synchronized (LOCK) { return _changeCallback; |