Menu
▾
▴
[Hyperic-hq-discuss] hq r14100 - in branches/HQ_4_2_0_PATCH: src/org/hyperic/hq/agent/server src/org/hyperic/hq/measurement/agent/server src/org/hyperic/util/file unittest/src/org/hyperic/util unittest/src/org/hyperic/util/file
Author: trader Date: 2009-12-16 17:12:32 -0800 (Wed, 16 Dec 2009) New Revision: 14100 URL: http://svn.hyperic.org/?view=rev&root=Hyperic+HQ&revision=14100 Added: branches/HQ_4_2_0_PATCH/unittest/src/org/hyperic/util/file/ branches/HQ_4_2_0_PATCH/unittest/src/org/hyperic/util/file/DiskListTest.java Modified: branches/HQ_4_2_0_PATCH/src/org/hyperic/hq/agent/server/AgentDListProvider.java branches/HQ_4_2_0_PATCH/src/org/hyperic/hq/measurement/agent/server/SenderThread.java branches/HQ_4_2_0_PATCH/src/org/hyperic/util/file/DiskList.java Log: HHQ-3541, 4.2.0 patch Modified: branches/HQ_4_2_0_PATCH/src/org/hyperic/hq/agent/server/AgentDListProvider.java =================================================================== --- branches/HQ_4_2_0_PATCH/src/org/hyperic/hq/agent/server/AgentDListProvider.java 2009-12-16 23:59:51 UTC (rev 14099) +++ branches/HQ_4_2_0_PATCH/src/org/hyperic/hq/agent/server/AgentDListProvider.java 2009-12-17 01:12:32 UTC (rev 14100) @@ -51,8 +51,8 @@ implements AgentStorageProvider { private static final int RECSIZE = 1024; - private static final long MAXSIZE = 100 * 1024 * 1024; // 100MB - private static final long CHKSIZE = 20 * 1024 * 1024; // 20MB + private static final long MAXSIZE = 50 * 1024 * 1024; // 50MB + private static final long CHKSIZE = 10 * 1024 * 1024; // 10MB private static final int CHKPERC = 50; // Only allow < 50% free private Log log; // da logger Modified: branches/HQ_4_2_0_PATCH/src/org/hyperic/hq/measurement/agent/server/SenderThread.java =================================================================== --- branches/HQ_4_2_0_PATCH/src/org/hyperic/hq/measurement/agent/server/SenderThread.java 2009-12-16 23:59:51 UTC (rev 14099) +++ branches/HQ_4_2_0_PATCH/src/org/hyperic/hq/measurement/agent/server/SenderThread.java 2009-12-17 01:12:32 UTC (rev 14100) @@ -379,9 +379,9 @@ { try { Record r = SenderThread.decodeRecord((String)it.next()); - if (!records.contains(r)) { - records.add(r); - } else { + boolean didNotAlreadyExist = records.add(r); + if (!didNotAlreadyExist) { + // nuke the dup if (debug) log.debug("Dropping duplicate entry for " + r); numUsed--; } @@ -607,6 +607,10 @@ return; } + if (log.isDebugEnabled()) { + log.debug("Woke up, sending batch of metrics."); + } + lastMetricTime = this.sendBatch(); if(lastMetricTime != null){ String backlogNum = ""; Modified: branches/HQ_4_2_0_PATCH/src/org/hyperic/util/file/DiskList.java =================================================================== --- branches/HQ_4_2_0_PATCH/src/org/hyperic/util/file/DiskList.java 2009-12-16 23:59:51 UTC (rev 14099) +++ branches/HQ_4_2_0_PATCH/src/org/hyperic/util/file/DiskList.java 2009-12-17 01:12:32 UTC (rev 14100) @@ -74,13 +74,14 @@ private static final Log log = LogFactory.getLog(DiskList.class.getName()); private String fileName; + private String idxFileName; private RandomAccessFile indexFile; - private RandomAccessFile dataFile; + protected RandomAccessFile dataFile; private int recordSize; // Size of each record private long firstRec; // IDX of first record private long lastRec; // IDX of last record private byte[] padBytes; // Utility array for padding - private SortedSet freeList; // Set(Long) of free rec idxs + protected SortedSet freeList; // Set(Long) of free rec idxs private int modNum; // Modification random number private long checkSize; // Start to check for unused blocks // when the datafile reaches this @@ -119,10 +120,11 @@ int checkPerc, long maxLength) throws IOException { - File idxFileName; + File idxFile; - idxFileName = new File(dataFile + ".idx"); + idxFile = new File(dataFile + ".idx"); this.fileName = dataFile.getName(); + this.idxFileName = idxFile.getName(); this.rand = new Random(); this.dataFile = new RandomAccessFile(dataFile, "rw"); this.recordSize = recordSize; @@ -135,9 +137,9 @@ " to " + maxLength + " bytes"); this.maxLength = maxLength; - this.genFreeList(idxFileName); + this.indexFile = new RandomAccessFile(idxFile, "rw"); + this.genFreeList(idxFile); - this.indexFile = new RandomAccessFile(idxFileName, "rw"); this.closed = false; } @@ -155,16 +157,16 @@ } /** - * Do maintinece on the data and index files. If the datafile size and - * the free block percentange exceed the defined thresholds, the extra + * Do maintenance on the data and index files. If the datafile size and + * the free block percentage exceed the defined thresholds, the extra * free blocks will be removed by truncating the data and index files. * * Since truncation is used, some times it will be possible that even - * though the criteria are met, we won't be albe to delete the free space. + * though the criteria are met, we won't be able to delete the free space. * This is a recoverable situation though, since new blocks will be * inserted at the beginning of the data file. */ - private void doMaintainence() + private void doMaintenence() throws IOException { long lastData = this.dataFile.length()/this.recordSize; @@ -211,7 +213,7 @@ * buffered input stream, which makes our initial startup much * faster, if there is a lot of data sitting in the list. */ - private void genFreeList(File idxFileName) + private void genFreeList(File idxFile) throws IOException { BufferedInputStream bIs; @@ -226,12 +228,12 @@ this.freeList = new TreeSet(); try { - fIs = new FileInputStream(idxFileName); + fIs = new FileInputStream(idxFile); bIs = new BufferedInputStream(fIs); dIs = new DataInputStream(bIs); - for(int idx=0; ; idx++){ + for(long idx=0; ; idx++){ boolean used; long prev, next; @@ -388,12 +390,20 @@ try { this.indexFile.setLength(0); } catch(IOException exc){ + this.log.error("IOException while truncating file " + idxFileName); + if (this.log.isDebugEnabled()) { + this.log.debug(exc); + } sExc = exc; } try { this.dataFile.setLength(0); } catch(IOException exc){ + this.log.error("IOException while truncating file " + fileName); + if (this.log.isDebugEnabled()) { + this.log.debug(exc); + } if(sExc != null){ sExc = exc; } @@ -469,9 +479,11 @@ this.freeList.add(new Long(recNo)); } - if ((this.dataFile.length() > this.checkSize) && - (this.getDataFileFreePercentage() > this.checkPerc)) { - this.doMaintainence(); + long length = this.dataFile.length(); + long percFree = this.getDataFileFreePercentage(); + if ((length > this.checkSize) && + (percFree > this.checkPerc)) { + this.doMaintenence(); } } @@ -492,13 +504,21 @@ try { this.dataFile.close(); - } catch(IOException exc){ + } catch(IOException exc){ + this.log.error("IOException while closing file " + fileName); + if (this.log.isDebugEnabled()) { + this.log.debug(exc); + } sExc = exc; } try { this.indexFile.close(); } catch(IOException exc){ + this.log.error("IOException while closing file " + idxFileName); + if (this.log.isDebugEnabled()) { + this.log.debug(exc); + } if(sExc == null){ sExc = exc; } @@ -550,6 +570,10 @@ try { rec = this.diskList.readRecord(this.curIdx); } catch(IOException exc){ + log.error("IOException while reading record"); + if (log.isDebugEnabled()) { + log.debug(exc); + } throw new NoSuchElementException("Error getting next " + "element: " + exc.getMessage()); @@ -577,6 +601,10 @@ try { this.diskList.removeRecord(this.curIdx); } catch(IOException exc){ + log.error("IOException while removing record"); + if (log.isDebugEnabled()) { + log.debug(exc); + } throw new IllegalStateException("Error removing record: " + exc.getMessage()); } @@ -597,7 +625,7 @@ return new DiskListIterator(this, this.firstRec, this.modNum); } } - + public static void main(String[] args) throws Exception { Added: branches/HQ_4_2_0_PATCH/unittest/src/org/hyperic/util/file/DiskListTest.java =================================================================== --- branches/HQ_4_2_0_PATCH/unittest/src/org/hyperic/util/file/DiskListTest.java (rev 0) +++ branches/HQ_4_2_0_PATCH/unittest/src/org/hyperic/util/file/DiskListTest.java 2009-12-17 01:12:32 UTC (rev 14100) @@ -0,0 +1,438 @@ +package org.hyperic.util.file; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.Iterator; + +import junit.framework.TestCase; + +public class DiskListTest extends TestCase +{ + private static final int RECSIZE = 1024; + private static final long MAXSIZE = 50 * 1024 * 1024; // 100MB + private static final long CHKSIZE = 10 * 1024 * 1024; // 20MB + private static final int CHKPERC = 50; // Only allow < 50% free + private static final int MAXRECS = 2000; + + public DiskListTest() {} + + public void setUp() { + } + + public void testReadWriteFile() throws Exception { + + DiskListDataHolder holder = null; + + try { + + try { + holder = new DiskListDataHolder(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.toString()); + } + + for (long i = 0; i < MAXRECS; ++i) { + String toPut = String.valueOf(i); + holder.list.addToList(toPut); + } + + Iterator it = holder.list.getListIterator(); + // Check that we can read the proper number of records back + long i = 0; + while (it.hasNext()) { + it.next(); + i++; + } + assertTrue(i == MAXRECS); + + holder.list.close(); + + // Check that we can read the proper number after close/reopen, and that they can be cleanly deleted + holder.list = new DiskList(holder.dataFile, + RECSIZE, + CHKSIZE, + CHKPERC, + MAXSIZE); + it = holder.list.getListIterator(); + i = 0; + while (it.hasNext()) { + it.next(); + it.remove(); + + i++; + } + + assertTrue(i == MAXRECS); + + } finally { + + holder.dispose(); + + } + } + + public void testFreeListWithNoInserts() throws Exception { + + DiskListDataHolder holder = null; + + try { + + try { + holder = new DiskListDataHolder(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.toString()); + } + + holder.list.close(); + + // Check that we can read the proper number after close/reopen, and that they can be cleanly deleted + holder.list = new DiskList(holder.dataFile, + RECSIZE, + CHKSIZE, + CHKPERC, + MAXSIZE); + + assertTrue(holder.list.freeList.size() == 0); + + } finally { + + holder.dispose(); + + } + } + + public void testFillAndReopen() throws Exception { + + DiskListDataHolder holder = null; + + try { + + try { + holder = new DiskListDataHolder(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.toString()); + } + + String toPut = String.valueOf("dummystring"); + + // Insert until we *almost* spill over + long nRecs = 0; + while (holder.list.dataFile.length() < MAXSIZE) { + holder.list.addToList(toPut); + nRecs++; + } + + holder.list.close(); + + // Check that we can read the proper number after close/reopen, and that they can be cleanly deleted + holder.list = new DiskList(holder.dataFile, + RECSIZE, + CHKSIZE, + CHKPERC, + MAXSIZE); + + assertTrue(holder.list.freeList.size() == 0); + + Iterator it = holder.list.getListIterator(); + long nIterated = 0; + while (it.hasNext()) { + it.next(); + nIterated++; + } + + assertTrue("Expected " + nRecs + " records, got " + nIterated, + nIterated == nRecs); + + } finally { + + holder.dispose(); + + } + } + + public void testFreeListWithInsertsAndNoDeletes() throws Exception { + + DiskListDataHolder holder = null; + if (holder == null) return; + + try { + + try { + holder = new DiskListDataHolder(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.toString()); + } + + for (long i = 0; i < MAXRECS; ++i) { + String toPut = String.valueOf(i); + holder.list.addToList(toPut); + } + + holder.list.close(); + + // Check that we can read the proper number after close/reopen, and that they can be cleanly deleted + holder.list = new DiskList(holder.dataFile, + RECSIZE, + CHKSIZE, + CHKPERC, + MAXSIZE); + + assertTrue(holder.list.freeList.size() == 0); + + } finally { + + holder.dispose(); + + } + } + + public void testFreeListWithInsertsAndDeletes() throws Exception { + + DiskListDataHolder holder = null; + + try { + + try { + holder = new DiskListDataHolder(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.toString()); + } + + for (long i = 0; i < MAXRECS; ++i) { + String toPut = String.valueOf(i); + holder.list.addToList(toPut); + } + + holder.list.close(); + + // Check that we can read the proper number after close/reopen, and that they can be cleanly deleted + holder.list = new DiskList(holder.dataFile, + RECSIZE, + CHKSIZE, + CHKPERC, + MAXSIZE); + Iterator it = holder.list.getListIterator(); + int nDeleted = 0; + while (it.hasNext()) { + for (int i = 0; i < 5; ++i) { + it.next(); + } + it.remove(); + nDeleted++; + } + + assertTrue(holder.list.freeList.size() == nDeleted); + + } finally { + + holder.dispose(); + + } + } + + public void testFreeListAfterTruncation() throws Exception { + + DiskListDataHolder holder = null; + long epsilon = 50; + + try { + + try { + holder = new DiskListDataHolder(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.toString()); + } + + String toPut = String.valueOf("dummystring"); + + // Insert until we *almost* spill over + while (holder.list.dataFile.length() + epsilon < MAXSIZE) { + holder.list.addToList(toPut); + } + + assertTrue(holder.list.freeList.size() == 0); + + // Now insert until we spill over, expect a log message from this line + while (holder.list.dataFile.length() > MAXSIZE - epsilon) { + holder.list.addToList(toPut); + } + + assertTrue(holder.list.freeList.size() == 0); + + // After truncation, there should be no records, add one back and then delete it + holder.list.addToList(toPut); + Iterator it = holder.list.getListIterator(); + int nIterated = 0; + while (it.hasNext()) { + it.next(); + + // Each remove should create one spot in the free list + it.remove(); + nIterated++; + } + + assertTrue(nIterated == 1); + assertTrue(holder.list.freeList.size() == 1); + + } finally { + + holder.dispose(); + + } + } + + public void testFreeListAfterMaintenance() throws Exception { + + DiskListDataHolder holder = null; + long epsilon = 50; + + try { + + try { + holder = new DiskListDataHolder(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.toString()); + } + + String toPut = String.valueOf("dummystring"); + + // Insert until we *almost* spill over + long nInserted = 0; + while (holder.list.dataFile.length() + epsilon < MAXSIZE) { + holder.list.addToList(toPut); + nInserted++; + } + + int freeListSize = holder.list.freeList.size(); + assertTrue(freeListSize == 0); + + // Delete every fourth record as we buzz through the list. Because CHKPERC is + // 50%, deleting every fourth should NOT trigger maintenance + int nIterated = 0; + int nDeleted = 0; + int counter = 0; + Iterator it = holder.list.getListIterator(); + while (it.hasNext()) { + it.next(); + + if (++counter == 4) { + // Each remove should create one spot in the free list + it.remove(); + nDeleted++; + counter = 0; + } + + nIterated++; + } + + assertTrue(nIterated == nInserted); + freeListSize = holder.list.freeList.size(); + assertTrue(freeListSize == nDeleted); + + // Now delete the the same amount - 1: still should NOT trigger maintenance + int nToDelete = nDeleted - 20; + int nPreviouslyDeleted = nDeleted; + nDeleted = 0; + it = holder.list.getListIterator(); + while (it.hasNext() && nDeleted < nToDelete) { + it.next(); + it.remove(); + nDeleted++; + } + + assertTrue(nDeleted == nToDelete); + freeListSize = holder.list.freeList.size(); + assertTrue(nDeleted + nPreviouslyDeleted == freeListSize); + + // Now try to trigger maintenance. First: maintenance is only done if the last block is + // free (maintenance truncates off the end only, no internal compacting), so make sure that + // there is stuff to truncate + int offTheEnd = 20; + it = holder.list.getListIterator(); + for (int i = nDeleted + nPreviouslyDeleted; i < nInserted - offTheEnd; ++i) { + it.next(); + } + while (it.hasNext()) { + it.next(); + it.remove(); + } + + // A few more deletes should trigger maintenance. The calculation uses an integer + // percentage rounded from doubles, so exactly (half - 1) deletes may not be enough. + freeListSize = holder.list.freeList.size(); + int freeListPeakSize = freeListSize; + long oldLength = holder.list.dataFile.length(); + int toTriggerMaintenance = ((int) (nInserted / 100)) / 2 + 1; + it = holder.list.getListIterator(); + for (int i = 0; i < toTriggerMaintenance; ++i) { + it.next(); + it.remove(); + freeListPeakSize = Math.max(freeListPeakSize, holder.list.freeList.size()); + } + + freeListSize = holder.list.freeList.size(); + long length = holder.list.dataFile.length(); + assertTrue("Expected free list size < " + freeListPeakSize + ", actual value was " + freeListSize, + freeListSize < freeListPeakSize); + assertTrue("Expected file length < " + oldLength + ", actual value was " + length, + length < oldLength); + + } finally { + + holder.dispose(); + + } + } + + private static class DiskListDataHolder { + DiskList list; + File dataFile; + File indexFile; + + DiskListDataHolder() throws Exception { + File tmpDirFile; + + String tmpDir = System.getProperty("java.io.tmpdir"); + if (tmpDir == null) { + tmpDir = "/tmp"; + } + + tmpDirFile = new File(tmpDir); + + File dataFile = null; + File indexFile = null; + if (tmpDirFile.isDirectory() && tmpDirFile.canWrite()) { + dataFile = new File(tmpDirFile, "datafile"); + indexFile = new File(tmpDirFile, "datafile.idx"); + dataFile.delete(); + indexFile.delete(); + } else { + throw new IllegalStateException("Non-writeable directory!"); + } + + DiskList list = new DiskList(dataFile, + RECSIZE, + CHKSIZE, + CHKPERC, + MAXSIZE); + + this.list = list; + this.dataFile = dataFile; + this.indexFile = indexFile; + } + + void dispose() throws Exception { + list.close(); + dataFile.delete(); + indexFile.delete(); + } + } +} |