[Archive-access-cvs] SF.net SVN: archive-access:[3531] trunk/archive-access/projects/wayback/ wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines

[<bra...@us...>]

Revision: 3531
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3531&view=rev
Author:   bradtofel
Date:     2011-09-06 04:18:00 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
INITIAL REV: BlockLoader which loads from Local RandomAccessFiles, and a generic BlockLoader which tries to do "the right thing" with each location, choosing the correct block loader based on the URL/String prefix

Added Paths:
-----------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/GenericBlockLoader.java
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/LocalFileBlockLoader.java

Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/GenericBlockLoader.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/GenericBlockLoader.java	                        (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/GenericBlockLoader.java	2011-09-06 04:18:00 UTC (rev 3531)
@@ -0,0 +1,78 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.wayback.resourceindex.ziplines;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+/**
+ * Generic BlockLoader, which may simplify configuration - inspecting each 
+ * location to attempt to choose the correct BlockLoader:
+ *   HDFS, HTTP, or LocalFile
+ * @author brad
+ *
+ */
+public class GenericBlockLoader implements BlockLoader {
+	Http11BlockLoader http = null;
+	HDFSBlockLoader hdfs = null;
+	LocalFileBlockLoader local = null;
+	private String defaultFSURI;
+	public GenericBlockLoader() {
+		http = new Http11BlockLoader();
+//		hdfs = new HDFSBlockLoader(null);
+//		try {
+//			hdfs.init();
+//		} catch (IOException e) {
+//			// TODO Auto-generated catch block
+//			e.printStackTrace();
+//		} catch (URISyntaxException e) {
+//			// TODO Auto-generated catch block
+//			e.printStackTrace();
+//		}
+		local = new LocalFileBlockLoader();
+	}
+	public void init() {
+		if(defaultFSURI != null) {
+			hdfs = new HDFSBlockLoader(defaultFSURI);
+			try {
+				hdfs.init();
+			} catch (IOException e) {
+				// TODO Auto-generated catch block
+				e.printStackTrace();
+			} catch (URISyntaxException e) {
+				// TODO Auto-generated catch block
+				e.printStackTrace();
+			}
+		}
+	}
+	public byte[] getBlock(String url, long offset, int length)
+			throws IOException {
+		if(hdfs != null && url.startsWith("hdfs://")) {
+			return hdfs.getBlock(url, offset, length);
+		} else if(url.startsWith("/")) {
+			return local.getBlock(url, offset, length);
+		}
+		return http.getBlock(url, offset, length);
+	}
+	public void setDefaultFSURI(String uri) {
+		defaultFSURI = uri;
+//		hdfs.setDefaultFSURI(uri);
+	}
+}

Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/LocalFileBlockLoader.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/LocalFileBlockLoader.java	                        (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/LocalFileBlockLoader.java	2011-09-06 04:18:00 UTC (rev 3531)
@@ -0,0 +1,47 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.wayback.resourceindex.ziplines;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+/**
+ * Simple block loader which uses RandomAccessFiles to grab ranges of local 
+ * files.
+ * @author brad
+ *
+ */
+public class LocalFileBlockLoader implements BlockLoader {
+
+	public byte[] getBlock(String url, long offset, int length)
+			throws IOException {
+		File file = new File(url);
+		RandomAccessFile raf = new RandomAccessFile(file, "r");
+		raf.seek(offset);
+		if(raf.getFilePointer() != offset) {
+			throw new IOException("Failed seek("+offset+") in ("+url+")");
+		}
+		byte b[] = new byte[length];
+		raf.readFully(b);
+		return b;
+	}
+
+}

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.