archive-access-cvs

[Brad <bra...@us...>]

Update of /cvsroot/archive-access/archive-access/projects/wayback/src/java/org/archive/wayback/localbdbresourceindex
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv14129/src/java/org/archive/wayback/localbdbresourceindex

Modified Files:
	LocalBDBResourceIndex.java BDBResourceIndex.java 
Removed Files:
	BDBResourceIndexWriter.java 
Log Message:
Heavy modification of configuration to be Context-level, instead of Servlet-level, which dramatically reduces configuration redundancy.

Cleaned up IndexPipeline, moved a few classes around, added a really simple JSP to view the Index and Merge queue sizes, and a filter, which both allows the index thread to start with the context, and allows access to the jsp.

Index: BDBResourceIndex.java
===================================================================
RCS file: /cvsroot/archive-access/archive-access/projects/wayback/src/java/org/archive/wayback/localbdbresourceindex/BDBResourceIndex.java,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** BDBResourceIndex.java	19 Oct 2005 01:22:37 -0000	1.2
--- BDBResourceIndex.java	21 Oct 2005 03:24:40 -0000	1.3
***************
*** 95,99 ****
  	}
  
! 	protected void shutdownDB() throws DatabaseException {
  
  		if (db != null) {
--- 95,104 ----
  	}
  
! 	/**
! 	 * shut down the BDB.
! 	 * 
! 	 * @throws DatabaseException
! 	 */
! 	public void shutdownDB() throws DatabaseException {
  
  		if (db != null) {
***************
*** 194,198 ****
  	}
  
! 	protected void addResults(ResourceResults results) throws Exception {
  		Iterator itr = results.iterator();
  		DatabaseEntry key = new DatabaseEntry();
--- 199,208 ----
  	}
  
! 	/**
! 	 * Add all ResourceResult in results to BDB index
! 	 * @param results
! 	 * @throws Exception
! 	 */
! 	public void addResults(ResourceResults results) throws Exception {
  		Iterator itr = results.iterator();
  		DatabaseEntry key = new DatabaseEntry();

--- BDBResourceIndexWriter.java DELETED ---

Index: LocalBDBResourceIndex.java
===================================================================
RCS file: /cvsroot/archive-access/archive-access/projects/wayback/src/java/org/archive/wayback/localbdbresourceindex/LocalBDBResourceIndex.java,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** LocalBDBResourceIndex.java	19 Oct 2005 01:22:37 -0000	1.2
--- LocalBDBResourceIndex.java	21 Oct 2005 03:24:40 -0000	1.3
***************
*** 24,28 ****
  package org.archive.wayback.localbdbresourceindex;
  
- import java.io.File;
  import java.io.IOException;
  import java.util.Properties;
--- 24,27 ----
***************
*** 42,60 ****
   */
  public class LocalBDBResourceIndex implements ResourceIndex {
- 	private static Thread indexUpdateThread = null;
  
! 	private final static String INDEX_PATH = "resourceindex.indexPath";
! 
! 	private final static String DB_NAME = "resourceindex.dbName";
! 
! 	private final static String ARC_PATH = "resourceindex.arcPath";
! 
! 	private final static String WORK_PATH = "resourceindex.workPath";
  
! 	private final static String RUN_PIPELINE = "resourceindex.runPipeline";
  
  	private final static int MAX_RECORDS = 1000;
  
  	private BDBResourceIndex db = null;
  
  	/**
--- 41,54 ----
   */
  public class LocalBDBResourceIndex implements ResourceIndex {
  
! 	private final static String INDEX_PATH = "resourceindex.indexpath";
  
! 	private final static String DB_NAME = "resourceindex.dbname";
  
  	private final static int MAX_RECORDS = 1000;
  
  	private BDBResourceIndex db = null;
+ 	
+ 	private IndexPipeline pipeline = null;
  
  	/**
***************
*** 71,100 ****
  			throw new IllegalArgumentException("Failed to find " + INDEX_PATH);
  		}
- 		String arcPath = (String) p.get(ARC_PATH);
- 		if (arcPath == null || (arcPath.length() <= 0)) {
- 			throw new IllegalArgumentException("Failed to find " + ARC_PATH);
- 		}
- 
- 		String workPath = (String) p.get(WORK_PATH);
- 		if (workPath == null || (workPath.length() <= 0)) {
- 			throw new IllegalArgumentException("Failed to find " + WORK_PATH);
- 		}
  		String dbName = (String) p.get(DB_NAME);
  		if (dbName == null || (dbName.length() <= 0)) {
  			throw new IllegalArgumentException("Failed to find " + DB_NAME);
  		}
! 		String runPipeline = (String) p.get(RUN_PIPELINE);
! 		db = new BDBResourceIndex(dbPath, dbName);
! 		if (runPipeline != null) {
! 
! 			System.out
! 					.println("LocalDBDResourceIndex starting pipeline thread...");
! 			if (indexUpdateThread == null) {
! 				IndexPipeline pipeline = new IndexPipeline();
! 				String mergeDir = workPath + "/mergey";
! 				pipeline.init(arcPath, mergeDir, workPath);
! 				startIndexUpdateThead(db, pipeline);
! 			}
! 		}
  	}
  
--- 65,75 ----
  			throw new IllegalArgumentException("Failed to find " + INDEX_PATH);
  		}
  		String dbName = (String) p.get(DB_NAME);
  		if (dbName == null || (dbName.length() <= 0)) {
  			throw new IllegalArgumentException("Failed to find " + DB_NAME);
  		}
! 		db = new BDBResourceIndex(dbPath,dbName);
! 		pipeline = new IndexPipeline();
! 		pipeline.init(p);
  	}
  
***************
*** 122,215 ****
  		}
  	}
- 
- 	protected synchronized void startIndexUpdateThead(
- 			final BDBResourceIndex bdb, IndexPipeline pipeline) {
- 		if (indexUpdateThread != null) {
- 			return;
- 		}
- 		indexUpdateThread = new IndexUpdateThread(bdb, pipeline);
- 		indexUpdateThread.start();
- 	}
- 
- 	/**
- 	 * Thread that repeatedly runs processing of an IndexPipeline and merges new
- 	 * data into a BDBResourceIndex
- 	 * 
- 	 * @author Brad Tofel
- 	 * @version $Date$, $Revision$
- 	 */
- 	private class IndexUpdateThread extends Thread {
- 		private final static int SLEEP_MILLISECONDS = 10000;
- 
- 		BDBResourceIndexWriter importer = null;
- 
- 		IndexPipeline pipeline = null;
- 
- 		/**
- 		 * Constructor
- 		 * 
- 		 * @param bdb
- 		 *            initialized BDBResourceIndex
- 		 * @param pipeline
- 		 *            initialized IndexPipeline
- 		 */
- 		public IndexUpdateThread(final BDBResourceIndex bdb,
- 				IndexPipeline pipeline) {
- 			super("IndexUpdateThread");
- 			super.setDaemon(true);
- 			this.importer = new BDBResourceIndexWriter();
- 			importer.init(bdb);
- 			this.pipeline = pipeline;
- 		}
- 
- 		public void run() {
- 
- 			while (true) {
- 				try {
- 					indexArcs();
- 					mergeIndex();
- 					sleep(SLEEP_MILLISECONDS);
- 				} catch (InterruptedException e) {
- 					e.printStackTrace();
- 				}
- 				// System.out.println("I'm running!");
- 			}
- 		}
- 
- 		private void indexArcs() {
- 			try {
- 				pipeline.indexArcs();
- 				// System.out.println("Indexed...");
- 			} catch (IOException e) {
- 				e.printStackTrace();
- 			}
- 		}
- 
- 		private void mergeIndex() {
- 			int numMerged = 0;
- 			String newFiles[] = pipeline.getMergeDir().list();
- 			for (int i = 0; i < newFiles.length; i++) {
- 				// TODO: Special handling of encoding and date.
- 				File newFile = new File(pipeline.getMergeDir()
- 						.getAbsolutePath()
- 						+ "/" + newFiles[i]);
- 
- 				if (newFile.isFile()) {
- 					try {
- 						importer.importFile(newFile.getAbsolutePath());
- 						if (!newFile.delete()) {
- 							throw new IOException("Unable to unlink "
- 									+ newFile.getAbsolutePath());
- 						}
- 						numMerged++;
- 					} catch (Exception e) {
- 						e.printStackTrace();
- 					}
- 				}
- 			}
- 			if (numMerged > 0) {
- 				System.out.println("Merged " + numMerged + " files.");
- 			}
- 		}
- 	}
  }
--- 97,99 ----