Menu
▾
▴
[Hyperic-hq-discuss] hq r14079 - in trunk: src/org/hyperic/hq/agent/server src/org/hyperic/hq/measurement/agent/server src/org/hyperic/util/file unittest/src/org/hyperic/util
Author: trader Date: 2009-12-11 16:16:21 -0800 (Fri, 11 Dec 2009) New Revision: 14079 URL: http://svn.hyperic.org/?view=rev&root=Hyperic+HQ&revision=14079 Added: trunk/unittest/src/org/hyperic/util/file/ Modified: trunk/src/org/hyperic/hq/agent/server/AgentDListProvider.java trunk/src/org/hyperic/hq/measurement/agent/server/SenderThread.java trunk/src/org/hyperic/util/file/DiskList.java Log: HHQ-3541, HHQ-3114: Integer overflow in DiskList, backlogged agent may throw OOM Clean up DiskList: better error logging, fix int overflow, add unittests. Modified: trunk/src/org/hyperic/hq/agent/server/AgentDListProvider.java =================================================================== --- trunk/src/org/hyperic/hq/agent/server/AgentDListProvider.java 2009-12-11 21:47:16 UTC (rev 14078) +++ trunk/src/org/hyperic/hq/agent/server/AgentDListProvider.java 2009-12-12 00:16:21 UTC (rev 14079) @@ -51,8 +51,8 @@ implements AgentStorageProvider { private static final int RECSIZE = 1024; - private static final long MAXSIZE = 100 * 1024 * 1024; // 100MB - private static final long CHKSIZE = 20 * 1024 * 1024; // 20MB + private static final long MAXSIZE = 50 * 1024 * 1024; // 50MB + private static final long CHKSIZE = 10 * 1024 * 1024; // 10MB private static final int CHKPERC = 50; // Only allow < 50% free private Log log; // da logger Modified: trunk/src/org/hyperic/hq/measurement/agent/server/SenderThread.java =================================================================== --- trunk/src/org/hyperic/hq/measurement/agent/server/SenderThread.java 2009-12-11 21:47:16 UTC (rev 14078) +++ trunk/src/org/hyperic/hq/measurement/agent/server/SenderThread.java 2009-12-12 00:16:21 UTC (rev 14079) @@ -607,6 +607,10 @@ return; } + if (log.isDebugEnabled()) { + log.debug("Woke up, sending batch of metrics."); + } + lastMetricTime = this.sendBatch(); if(lastMetricTime != null){ String backlogNum = ""; Modified: trunk/src/org/hyperic/util/file/DiskList.java =================================================================== --- trunk/src/org/hyperic/util/file/DiskList.java 2009-12-11 21:47:16 UTC (rev 14078) +++ trunk/src/org/hyperic/util/file/DiskList.java 2009-12-12 00:16:21 UTC (rev 14079) @@ -74,13 +74,14 @@ private static final Log log = LogFactory.getLog(DiskList.class.getName()); private String fileName; + private String idxFileName; private RandomAccessFile indexFile; - private RandomAccessFile dataFile; + protected RandomAccessFile dataFile; private int recordSize; // Size of each record private long firstRec; // IDX of first record private long lastRec; // IDX of last record private byte[] padBytes; // Utility array for padding - private SortedSet freeList; // Set(Long) of free rec idxs + protected SortedSet freeList; // Set(Long) of free rec idxs private int modNum; // Modification random number private long checkSize; // Start to check for unused blocks // when the datafile reaches this @@ -119,10 +120,11 @@ int checkPerc, long maxLength) throws IOException { - File idxFileName; + File idxFile; - idxFileName = new File(dataFile + ".idx"); + idxFile = new File(dataFile + ".idx"); this.fileName = dataFile.getName(); + this.idxFileName = idxFile.getName(); this.rand = new Random(); this.dataFile = new RandomAccessFile(dataFile, "rw"); this.recordSize = recordSize; @@ -135,9 +137,9 @@ " to " + maxLength + " bytes"); this.maxLength = maxLength; - this.genFreeList(idxFileName); + this.indexFile = new RandomAccessFile(idxFile, "rw"); + this.genFreeList(idxFile); - this.indexFile = new RandomAccessFile(idxFileName, "rw"); this.closed = false; } @@ -155,16 +157,16 @@ } /** - * Do maintinece on the data and index files. If the datafile size and - * the free block percentange exceed the defined thresholds, the extra + * Do maintenance on the data and index files. If the datafile size and + * the free block percentage exceed the defined thresholds, the extra * free blocks will be removed by truncating the data and index files. * * Since truncation is used, some times it will be possible that even - * though the criteria are met, we won't be albe to delete the free space. + * though the criteria are met, we won't be able to delete the free space. * This is a recoverable situation though, since new blocks will be * inserted at the beginning of the data file. */ - private void doMaintainence() + private void doMaintenence() throws IOException { long lastData = this.dataFile.length()/this.recordSize; @@ -211,7 +213,7 @@ * buffered input stream, which makes our initial startup much * faster, if there is a lot of data sitting in the list. */ - private void genFreeList(File idxFileName) + private void genFreeList(File idxFile) throws IOException { BufferedInputStream bIs; @@ -226,12 +228,12 @@ this.freeList = new TreeSet(); try { - fIs = new FileInputStream(idxFileName); + fIs = new FileInputStream(idxFile); bIs = new BufferedInputStream(fIs); dIs = new DataInputStream(bIs); - for(int idx=0; ; idx++){ + for(long idx=0; ; idx++){ boolean used; long prev, next; @@ -388,12 +390,20 @@ try { this.indexFile.setLength(0); } catch(IOException exc){ + this.log.error("IOException while truncating file " + idxFileName); + if (this.log.isDebugEnabled()) { + this.log.debug(exc); + } sExc = exc; } try { this.dataFile.setLength(0); } catch(IOException exc){ + this.log.error("IOException while truncating file " + fileName); + if (this.log.isDebugEnabled()) { + this.log.debug(exc); + } if(sExc != null){ sExc = exc; } @@ -469,9 +479,11 @@ this.freeList.add(new Long(recNo)); } - if ((this.dataFile.length() > this.checkSize) && - (this.getDataFileFreePercentage() > this.checkPerc)) { - this.doMaintainence(); + long length = this.dataFile.length(); + long percFree = this.getDataFileFreePercentage(); + if ((length > this.checkSize) && + (percFree > this.checkPerc)) { + this.doMaintenence(); } } @@ -492,13 +504,21 @@ try { this.dataFile.close(); - } catch(IOException exc){ + } catch(IOException exc){ + this.log.error("IOException while closing file " + fileName); + if (this.log.isDebugEnabled()) { + this.log.debug(exc); + } sExc = exc; } try { this.indexFile.close(); } catch(IOException exc){ + this.log.error("IOException while closing file " + idxFileName); + if (this.log.isDebugEnabled()) { + this.log.debug(exc); + } if(sExc == null){ sExc = exc; } @@ -550,6 +570,10 @@ try { rec = this.diskList.readRecord(this.curIdx); } catch(IOException exc){ + log.error("IOException while reading record"); + if (log.isDebugEnabled()) { + log.debug(exc); + } throw new NoSuchElementException("Error getting next " + "element: " + exc.getMessage()); @@ -577,6 +601,10 @@ try { this.diskList.removeRecord(this.curIdx); } catch(IOException exc){ + log.error("IOException while removing record"); + if (log.isDebugEnabled()) { + log.debug(exc); + } throw new IllegalStateException("Error removing record: " + exc.getMessage()); } @@ -597,7 +625,7 @@ return new DiskListIterator(this, this.firstRec, this.modNum); } } - + public static void main(String[] args) throws Exception { |
