[Joafip-svn] SF.net SVN: joafip:[3169] trunk/joafip-4test
Brought to you by:
luc_peuvrier
From: <luc...@us...> - 2012-11-28 12:17:27
|
Revision: 3169 http://joafip.svn.sourceforge.net/joafip/?rev=3169&view=rev Author: luc_peuvrier Date: 2012-11-28 12:17:20 +0000 (Wed, 28 Nov 2012) Log Message: ----------- red black tree management changed. foreground garbage sweep changed Modified Paths: -------------- trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractPerfService.java trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractSearcher.java trunk/joafip-4test/src/main/resources/log4j.properties Added Paths: ----------- trunk/joafip-4test/src/main/java/net/sf/joafip/heapfile/service/ trunk/joafip-4test/src/main/java/net/sf/joafip/heapfile/service/CheckIdNodeColor.java trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractDeleteForGarbage.java trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractGarbageSweep.java trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/DeleteForGarbage.java trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/GarbageSweep.java Property Changed: ---------------- trunk/joafip-4test/ Property changes on: trunk/joafip-4test ___________________________________________________________________ Modified: svn:ignore - target .settings .classpath .project runtime logs .pmd lib runtime_perf jrat.output + target .settings .classpath .project runtime logs .pmd lib runtime_perf jrat.output runtime_garbage Added: trunk/joafip-4test/src/main/java/net/sf/joafip/heapfile/service/CheckIdNodeColor.java =================================================================== --- trunk/joafip-4test/src/main/java/net/sf/joafip/heapfile/service/CheckIdNodeColor.java (rev 0) +++ trunk/joafip-4test/src/main/java/net/sf/joafip/heapfile/service/CheckIdNodeColor.java 2012-11-28 12:17:20 UTC (rev 3169) @@ -0,0 +1,116 @@ +/* + * Copyright 2012 Luc Peuvrier + * All rights reserved. + * + * This file is a part of JOAFIP. + * + * JOAFIP is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License. + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE + * Licensed under the LGPL License, Version 3, 29 June 2007 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.gnu.org/licenses/lgpl.html + * + * JOAFIP is distributed in the hope that it will be useful, but + * unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.sf.joafip.heapfile.service; + +import java.io.File; +import java.util.Iterator; + +import net.sf.joafip.heapfile.record.entity.HeapIdNode; +import net.sf.joafip.kvstore.entity.EnumFileAccessMode; +import net.sf.joafip.kvstore.entity.HeapFileSetup; +import net.sf.joafip.kvstore.record.entity.DataRecordIdentifier; +import net.sf.joafip.kvstore.service.HeapException; +import net.sf.joafip.logger.JoafipLogger; +import net.sf.joafip.performance.items.service.AbstractPerfService; +import net.sf.joafip.redblacktree.entity.IRBTNode; +import net.sf.joafip.redblacktree.service.RedBlackTree; +import net.sf.joafip.store.entity.StoreProperties; +import net.sf.joafip.store.service.AbstractStoreDelegatingToDataManager; + +/** + * + * @author luc peuvrier + * + */ +public class CheckIdNodeColor { + + private static final JoafipLogger LOGGER = JoafipLogger + .getLogger(CheckIdNodeColor.class); + + public static void main(String[] args) { + try { + final CheckIdNodeColor main = new CheckIdNodeColor(); + main.run(); + } catch (Throwable throwable) { + LOGGER.fatal(throwable); + } + } + + private void run() throws HeapException { + + final StoreProperties storeProperties = new StoreProperties(); + + final EnumFileAccessMode fileAccessMode = EnumFileAccessMode.NIO_RANDOM_FILE_ACCESS; + + final String toUseDataFileName = AbstractStoreDelegatingToDataManager.STORE_DATA; + final String pathName = AbstractPerfService.RUNTIME_DIR; + final File dataFile = new File(pathName, toUseDataFileName); + + final boolean useCacheMode = true; + final boolean crashSafeMode = false; + + final boolean manageNodeIndex = false; + + final boolean deleteRenaming = storeProperties.isDeleteRenaming(); + final boolean clearResizeFile = storeProperties.isClearResizeFile(); + final int maxFileOperationRetry = storeProperties + .getMaxFileOperationRetry(); + final int fileOperationRetryMsDelay = storeProperties + .getFileOperationRetryMsDelay(); + + final File openFileTraceFile = new File(pathName, + AbstractStoreDelegatingToDataManager.TRACE_FILE); + + final HeapFileSetup setup = new HeapFileSetup(fileAccessMode, dataFile, + crashSafeMode, useCacheMode, deleteRenaming, clearResizeFile, + maxFileOperationRetry, fileOperationRetryMsDelay, + openFileTraceFile); + final int maxPage = AbstractPerfService.NUMBER_OF_PAGE; + final int pageSize = AbstractPerfService.PAGE_SIZE; + setup.cacheSetup(pageSize, maxPage); + HeapFileDataManager dataManager = new HeapFileDataManager(setup, + manageNodeIndex); + + dataManager.startService(false/* removeFiles */); + + RedBlackTree<DataRecordIdentifier> idNodeTree = dataManager + .getIdNodeTree(); + + Iterator<IRBTNode<DataRecordIdentifier>> iterator = idNodeTree + .iterator(); + int count = 0; + while (iterator.hasNext()) { + HeapIdNode heapIdNode = (HeapIdNode) iterator.next(); + if (!heapIdNode.isColorSetted()) { + LOGGER.error("color not set " + heapIdNode.getElement()); + } + count++; + if (count % 1000 == 999) { + dataManager.flush(); + } + } + dataManager.stopService(); + LOGGER.info("number of id record " + count); + } +} Property changes on: trunk/joafip-4test/src/main/java/net/sf/joafip/heapfile/service/CheckIdNodeColor.java ___________________________________________________________________ Added: svn:mime-type + text/plain Added: trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractDeleteForGarbage.java =================================================================== --- trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractDeleteForGarbage.java (rev 0) +++ trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractDeleteForGarbage.java 2012-11-28 12:17:20 UTC (rev 3169) @@ -0,0 +1,95 @@ +/* + * Copyright 2012 Luc Peuvrier + * All rights reserved. + * + * This file is a part of JOAFIP. + * + * JOAFIP is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License. + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE + * Licensed under the LGPL License, Version 3, 29 June 2007 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.gnu.org/licenses/lgpl.html + * + * JOAFIP is distributed in the hope that it will be useful, but + * unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.sf.joafip.performance.items.service; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; + +import net.sf.joafip.entity.EnumFilePersistenceCloseAction; +import net.sf.joafip.kvstore.service.IHeapDataManager; +import net.sf.joafip.performance.items.entity.Item; +import net.sf.joafip.performance.items.entity.ItemList; +import net.sf.joafip.service.FilePersistenceClassNotFoundException; +import net.sf.joafip.service.FilePersistenceDataCorruptedException; +import net.sf.joafip.service.FilePersistenceException; +import net.sf.joafip.service.FilePersistenceInvalidClassException; +import net.sf.joafip.service.FilePersistenceNotSerializableException; +import net.sf.joafip.service.FilePersistenceTooBigForSerializationException; +import net.sf.joafip.service.IDataAccessSession; + +/** + * + * @author luc peuvrier + * + */ +public class AbstractDeleteForGarbage extends AbstractPerfService { + + protected void run(final String insertLogFilePath, final String pathName, + final IHeapDataManager dataManager, final Boolean useCache) + throws FilePersistenceInvalidClassException, + FilePersistenceNotSerializableException, + FilePersistenceClassNotFoundException, + FilePersistenceDataCorruptedException, FilePersistenceException, + FilePersistenceTooBigForSerializationException, IOException { + initialize(pathName, dataManager, useCache); + logger.info("nbdr " + filePersistence.getNumberOfDataRecord()); + final InputStream inputStream = new FileInputStream(insertLogFilePath); + final BufferedReader reader = new BufferedReader(new InputStreamReader( + inputStream)); + final IDataAccessSession session = filePersistence + .createDataAccessSession(); + session.open(); + // delete half of item + ItemList itemList = getItemList(session); + int count = 0; + String line = reader.readLine(); + while (line != null) { // NOPMD + count++; + final int identifier = Integer.parseInt(line); + Item removed = itemList.removeItem(identifier); + if( removed==null){ + logger.error("not found "+identifier); + } + if (count % BATCH_SIZE == BATCH_SIZE - 1) { + session.close(EnumFilePersistenceCloseAction.SAVE); + session.open(); + itemList = getItemList(session); + } + line = reader.readLine(); + if (line != null) { + line = reader.readLine(); + } + } + session.close(EnumFilePersistenceCloseAction.SAVE); + reader.close(); + logger.info("nbdr " + filePersistence.getNumberOfDataRecord()); + } + + protected void close() throws FilePersistenceException { + filePersistence.close(); + } +} Property changes on: trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractDeleteForGarbage.java ___________________________________________________________________ Added: svn:mime-type + text/plain Added: trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractGarbageSweep.java =================================================================== --- trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractGarbageSweep.java (rev 0) +++ trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractGarbageSweep.java 2012-11-28 12:17:20 UTC (rev 3169) @@ -0,0 +1,67 @@ +/* + * Copyright 2012 Luc Peuvrier + * All rights reserved. + * + * This file is a part of JOAFIP. + * + * JOAFIP is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License. + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE + * Licensed under the LGPL License, Version 3, 29 June 2007 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.gnu.org/licenses/lgpl.html + * + * JOAFIP is distributed in the hope that it will be useful, but + * unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.sf.joafip.performance.items.service; + +import java.io.IOException; + +import net.sf.joafip.kvstore.service.IHeapDataManager; +import net.sf.joafip.service.FilePersistenceClassNotFoundException; +import net.sf.joafip.service.FilePersistenceDataCorruptedException; +import net.sf.joafip.service.FilePersistenceException; +import net.sf.joafip.service.FilePersistenceInvalidClassException; +import net.sf.joafip.service.FilePersistenceNotSerializableException; +import net.sf.joafip.service.FilePersistenceTooBigForSerializationException; + +/** + * + * @author luc peuvrier + * + */ +public class AbstractGarbageSweep extends AbstractPerfService { + + protected void run(final String pathName, + final IHeapDataManager dataManager, final Boolean useCache) + throws FilePersistenceInvalidClassException, + FilePersistenceNotSerializableException, + FilePersistenceClassNotFoundException, + FilePersistenceDataCorruptedException, FilePersistenceException, + FilePersistenceTooBigForSerializationException, IOException { + initialize(pathName, dataManager, useCache); + logger.info("nbdr after delete " + + filePersistence.getNumberOfDataRecord()); + + final long beginTime = System.currentTimeMillis(); + final int garbaged = filePersistence + .garbageSweep(pathName + "/garbage"); + final long endTime = System.currentTimeMillis(); + logger.info(garbaged + " garbaged, duration " + (endTime - beginTime) + + " mS"); + logger.info("nbdr after garbage " + + filePersistence.getNumberOfDataRecord()); + } + + protected void close() throws FilePersistenceException { + filePersistence.close(); + } +} \ No newline at end of file Property changes on: trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractGarbageSweep.java ___________________________________________________________________ Added: svn:mime-type + text/plain Modified: trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractPerfService.java =================================================================== --- trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractPerfService.java 2012-11-28 12:16:51 UTC (rev 3168) +++ trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractPerfService.java 2012-11-28 12:17:20 UTC (rev 3169) @@ -46,7 +46,7 @@ protected final JoafipLogger logger = JoafipLogger.getLogger(getClass());// NOPMD - protected static final String RUNTIME_DIR = "runtime_perf"; // "z:\\";// + public static final String RUNTIME_DIR = "runtime_perf"; // "z:\\";// public static final File DURATION_BIN = new File(RUNTIME_DIR + "/duration.bin"); @@ -59,13 +59,13 @@ private static final boolean GARBAGE = false; - protected static final int PAGE_SIZE = 4 * 1024; + public static final int PAGE_SIZE = 4 * 1024; - protected static final int NUMBER_OF_PAGE = 4 * 1024; + public static final int NUMBER_OF_PAGE = 4 * 1024; - private static final String ITEM_TEMPLATE_LIST = "itemTemplateList"; + protected static final String ITEM_TEMPLATE_LIST = "itemTemplateList"; - private static final String ITEM_LIST = "itemList"; + protected static final String ITEM_LIST = "itemList"; protected IFilePersistence filePersistence; @@ -91,9 +91,11 @@ final FilePersistenceBuilder builder = new FilePersistenceBuilder(); try { if (dataManager == null) { + builder.setFileAccessMode(EnumFileAccessMode.MAPPED_RANDOM_FILE_ACCESS); builder.setMaxBufferSize(512 * 1024); builder.setMaxNumberOfBuffer(256); + builder.setPathName(pathName); if (useCache) { builder.setFileCache(PAGE_SIZE, NUMBER_OF_PAGE); Modified: trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractSearcher.java =================================================================== --- trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractSearcher.java 2012-11-28 12:16:51 UTC (rev 3168) +++ trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/AbstractSearcher.java 2012-11-28 12:17:20 UTC (rev 3169) @@ -112,7 +112,7 @@ if (searchDuration < minSearchTime) { minSearchTime = searchDuration; } - if (identifier % BATCH_SIZE == BATCH_SIZE - 1) { + if (count % BATCH_SIZE == BATCH_SIZE - 1) { session.close(EnumFilePersistenceCloseAction.DO_NOT_SAVE); session.open(); final long duration = (currentTime - startTime); Added: trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/DeleteForGarbage.java =================================================================== --- trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/DeleteForGarbage.java (rev 0) +++ trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/DeleteForGarbage.java 2012-11-28 12:17:20 UTC (rev 3169) @@ -0,0 +1,68 @@ +/* + * Copyright 2012 Luc Peuvrier + * All rights reserved. + * + * This file is a part of JOAFIP. + * + * JOAFIP is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License. + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE + * Licensed under the LGPL License, Version 3, 29 June 2007 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.gnu.org/licenses/lgpl.html + * + * JOAFIP is distributed in the hope that it will be useful, but + * unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.sf.joafip.performance.items.service; + +import java.io.FileNotFoundException; +import java.io.IOException; + +import net.sf.joafip.logger.JoafipLogger; +import net.sf.joafip.service.FilePersistenceClassNotFoundException; +import net.sf.joafip.service.FilePersistenceDataCorruptedException; +import net.sf.joafip.service.FilePersistenceException; +import net.sf.joafip.service.FilePersistenceInvalidClassException; +import net.sf.joafip.service.FilePersistenceNotSerializableException; +import net.sf.joafip.service.FilePersistenceTooBigForSerializationException; + +/** + * + * @author luc peuvrier + * + */ +public class DeleteForGarbage extends AbstractDeleteForGarbage { + + private static final JoafipLogger LOGGER = JoafipLogger + .getLogger(DeleteForGarbage.class); + + private void run() throws FileNotFoundException, FilePersistenceException, + FilePersistenceClassNotFoundException, + FilePersistenceInvalidClassException, + FilePersistenceDataCorruptedException, + FilePersistenceNotSerializableException, IOException, + FilePersistenceTooBigForSerializationException { + final String insertLogFilePath = RUNTIME_DIR + "/inserted.txt"; + run(insertLogFilePath,RUNTIME_DIR, null, true); + close(); + } + + public static void main(final String[] args) { + DeleteForGarbage deleteAndGarbage; + try { + deleteAndGarbage = new DeleteForGarbage(); + deleteAndGarbage.run(); + } catch (final Throwable throwable) {// NOPMD catch all + LOGGER.fatal("error", throwable); + } + } + +} Property changes on: trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/DeleteForGarbage.java ___________________________________________________________________ Added: svn:mime-type + text/plain Added: trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/GarbageSweep.java =================================================================== --- trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/GarbageSweep.java (rev 0) +++ trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/GarbageSweep.java 2012-11-28 12:17:20 UTC (rev 3169) @@ -0,0 +1,78 @@ +/* + * Copyright 2012 Luc Peuvrier + * All rights reserved. + * + * This file is a part of JOAFIP. + * + * JOAFIP is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License. + * + * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE + * Licensed under the LGPL License, Version 3, 29 June 2007 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.gnu.org/licenses/lgpl.html + * + * JOAFIP is distributed in the hope that it will be useful, but + * unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.sf.joafip.performance.items.service; + +import java.io.File; +import java.io.IOException; + +import net.sf.joafip.file.service.FileIOException; +import net.sf.joafip.file.service.HelperFileUtil; +import net.sf.joafip.logger.JoafipLogger; +import net.sf.joafip.service.FilePersistenceClassNotFoundException; +import net.sf.joafip.service.FilePersistenceDataCorruptedException; +import net.sf.joafip.service.FilePersistenceException; +import net.sf.joafip.service.FilePersistenceInvalidClassException; +import net.sf.joafip.service.FilePersistenceNotSerializableException; +import net.sf.joafip.service.FilePersistenceTooBigForSerializationException; + +/** + * + * @author luc peuvrier + * + */ +public class GarbageSweep extends AbstractGarbageSweep { + + private final static JoafipLogger LOGGER = JoafipLogger + .getLogger(GarbageSweep.class); + + public static void main(String[] args) { + try { + final GarbageSweep main = new GarbageSweep(); + main.run(); + } catch (Exception exception) { + LOGGER.fatal(exception); + } + } + + private void run() throws FilePersistenceInvalidClassException, + FilePersistenceNotSerializableException, + FilePersistenceClassNotFoundException, + FilePersistenceDataCorruptedException, FilePersistenceException, + FilePersistenceTooBigForSerializationException, IOException, + FileIOException { + final String pathName = "runtime_garbage"; + final String sourcePathName = RUNTIME_DIR; + + (new File(pathName)).mkdirs(); + final File[] list = (new File(sourcePathName)).listFiles(); + for (File sourceFile : list) { + final File destinationFile = new File(pathName, + sourceFile.getName()); + HelperFileUtil.getInstance().copyFile(sourceFile, destinationFile); + } + + run(pathName, null, true); + close(); + } +} Property changes on: trunk/joafip-4test/src/main/java/net/sf/joafip/performance/items/service/GarbageSweep.java ___________________________________________________________________ Added: svn:mime-type + text/plain Modified: trunk/joafip-4test/src/main/resources/log4j.properties =================================================================== --- trunk/joafip-4test/src/main/resources/log4j.properties 2012-11-28 12:16:51 UTC (rev 3168) +++ trunk/joafip-4test/src/main/resources/log4j.properties 2012-11-28 12:17:20 UTC (rev 3169) @@ -143,6 +143,12 @@ log4j.logger.net.sf.joafip.performance.items.service.DeleterBtreePlus=info +log4j.logger.net.sf.joafip.performance.items.service.DeleteForGarbage=info +log4j.logger.net.sf.joafip.performance.items.service.GarbageSweep=info +log4j.logger.net.sf.joafip.store.service.garbage.StoreGarbager=info +log4j.logger.net.sf.joafip.heapfile.service.CheckIdNodeColor=info +log4j.logger.net.sf.joafip.redblacktree.service.TreeShower=info + log4j.logger.net.sf.joafip.service.MainCrash=info log4j.logger.net.sf.joafip.service.MainAfterCrash=info log4j.logger.net.sf.joafip.service.MainWarnDataRecord1=debug This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |