Thread: [Archive-access-cvs] SF.net SVN: archive-access:[3001] trunk/archive-access/projects/wayback/ wayb

Brought to you by: binzino, bradtofel, gojomo, ia_igor, and 5 others

archive-access-cvs

[Archive-access-cvs] SF.net SVN: archive-access:[3001] trunk/archive-access/projects/wayback/ wayback-core/src/main/java/org/archive/wayback/archivalurl/ ArchivalUrlSAXRewriteReplayRenderer.java

From: <bra...@us...> - 2010-03-23 23:46:19

Revision: 3001
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3001&view=rev
Author:   bradtofel
Date:     2010-03-23 23:46:12 +0000 (Tue, 23 Mar 2010)

Log Message:
-----------
BUGFIX(unreported): was not setting STRICT_REMARKS to false - causing problems with many web pages using <!--- ----> and such.

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlSAXRewriteReplayRenderer.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlSAXRewriteReplayRenderer.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlSAXRewriteReplayRenderer.java	2010-03-23 23:45:07 UTC (rev 3000)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlSAXRewriteReplayRenderer.java	2010-03-23 23:46:12 UTC (rev 3001)
@@ -123,8 +123,9 @@
 		// and finally, parse, using the special lexer that knows how to
 		// handle javascript blocks containing unescaped HTML entities:
 		Page lexPage = new Page(resource,charSet);
-    	ContextAwareLexer lex = new ContextAwareLexer(new Lexer(lexPage),
-    			context);
+		Lexer lexer = new Lexer(lexPage);
+		Lexer.STRICT_REMARKS = false;
+    	ContextAwareLexer lex = new ContextAwareLexer(lexer, context);
     	Node node;
     	try {
 			while((node = lex.nextNode()) != null) {


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Archive-access-cvs] SF.net SVN: archive-access:[3092] trunk/archive-access/projects/wayback/ wayback-core/src/main/java/org/archive/wayback/archivalurl/ ArchivalUrlSAXRewriteReplayRenderer.java

From: <bra...@us...> - 2010-05-17 19:26:18

Revision: 3092
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3092&view=rev
Author:   bradtofel
Date:     2010-05-17 19:26:12 +0000 (Mon, 17 May 2010)

Log Message:
-----------
TWEAK: changed declared event delgator to ParseEventHandler

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlSAXRewriteReplayRenderer.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlSAXRewriteReplayRenderer.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlSAXRewriteReplayRenderer.java	2010-05-14 22:52:50 UTC (rev 3091)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlSAXRewriteReplayRenderer.java	2010-05-17 19:26:12 UTC (rev 3092)
@@ -44,11 +44,12 @@
 import org.archive.wayback.replay.HttpHeaderOperation;
 import org.archive.wayback.replay.HttpHeaderProcessor;
 import org.archive.wayback.replay.JSPExecutor;
+import org.archive.wayback.replay.TextReplayRenderer;
 import org.archive.wayback.replay.charset.CharsetDetector;
 import org.archive.wayback.replay.charset.StandardCharsetDetector;
-import org.archive.wayback.replay.html.ReplayParseEventDelegator;
 import org.archive.wayback.replay.html.ReplayParseContext;
 import org.archive.wayback.util.htmllex.ContextAwareLexer;
+import org.archive.wayback.util.htmllex.ParseEventHandler;
 import org.htmlparser.Node;
 import org.htmlparser.lexer.Lexer;
 import org.htmlparser.lexer.Page;
@@ -62,7 +63,7 @@
  *
  */
 public class ArchivalUrlSAXRewriteReplayRenderer implements ReplayRenderer {
-	private ReplayParseEventDelegator delegator = null;
+	private ParseEventHandler delegator = null;
 	private HttpHeaderProcessor httpHeaderProcessor;
 	private CharsetDetector charsetDetector = new StandardCharsetDetector();
 	private final static String OUTPUT_CHARSET = "utf-8";
@@ -142,7 +143,7 @@
 		// set the corrected length:
 		headers.put(HttpHeaderOperation.HTTP_LENGTH_HEADER, 
 				String.valueOf(utf8Bytes.length));
-		headers.put("X-Wayback-Guessed-Charset", charSet);
+		headers.put(TextReplayRenderer.GUESSED_CHARSET_HEADER, charSet);
 
 		// send back the headers:
 		HttpHeaderOperation.sendHeaders(headers, httpResponse);
@@ -173,14 +174,14 @@
 	/**
 	 * @return the delegator
 	 */
-	public ReplayParseEventDelegator getDelegator() {
+	public ParseEventHandler getDelegator() {
 		return delegator;
 	}
 
 	/**
 	 * @param delegator the delegator to set
 	 */
-	public void setDelegator(ReplayParseEventDelegator delegator) {
+	public void setDelegator(ParseEventHandler delegator) {
 		this.delegator = delegator;
 	}
 }


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Archive-access-cvs] SF.net SVN: archive-access:[3451] trunk/archive-access/projects/wayback/ wayback-core/src/main/java/org/archive/wayback/archivalurl/ ArchivalUrlSAXRewriteReplayRenderer.java

From: <bra...@us...> - 2011-05-25 01:37:16

Revision: 3451
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3451&view=rev
Author:   bradtofel
Date:     2011-05-25 01:37:10 +0000 (Wed, 25 May 2011)

Log Message:
-----------
FEATURE: added FrameWrapped state tracking code

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlSAXRewriteReplayRenderer.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlSAXRewriteReplayRenderer.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlSAXRewriteReplayRenderer.java	2011-05-25 01:36:49 UTC (rev 3450)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlSAXRewriteReplayRenderer.java	2011-05-25 01:37:10 UTC (rev 3451)
@@ -19,10 +19,12 @@
  */
 package org.archive.wayback.archivalurl;
 
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.nio.charset.Charset;
 import java.util.Map;
 
 import javax.servlet.ServletException;
@@ -39,15 +41,19 @@
 import org.archive.wayback.replay.HttpHeaderOperation;
 import org.archive.wayback.replay.HttpHeaderProcessor;
 import org.archive.wayback.replay.JSPExecutor;
+import org.archive.wayback.replay.TagMagix;
 import org.archive.wayback.replay.TextReplayRenderer;
 import org.archive.wayback.replay.charset.CharsetDetector;
 import org.archive.wayback.replay.charset.StandardCharsetDetector;
 import org.archive.wayback.replay.html.ReplayParseContext;
+import org.archive.wayback.util.ByteOp;
 import org.archive.wayback.util.htmllex.ContextAwareLexer;
 import org.archive.wayback.util.htmllex.ParseEventHandler;
 import org.htmlparser.Node;
+import org.htmlparser.lexer.InputStreamSource;
 import org.htmlparser.lexer.Lexer;
 import org.htmlparser.lexer.Page;
+import org.htmlparser.lexer.Source;
 import org.htmlparser.util.ParserException;
 
 /**
@@ -62,7 +68,16 @@
 	private HttpHeaderProcessor httpHeaderProcessor;
 	private CharsetDetector charsetDetector = new StandardCharsetDetector();
 	private final static String OUTPUT_CHARSET = "utf-8";
+	private static int FRAMESET_SCAN_BUFFER_SIZE = 16 * 1024;
+	private static ReplayRenderer frameWrappingRenderer = null;
+	public static ReplayRenderer getFrameWrappingRenderer() {
+		return frameWrappingRenderer;
+	}
 
+	public static void setFrameWrappingRenderer(ReplayRenderer frameWrappingRenderer) {
+		ArchivalUrlSAXRewriteReplayRenderer.frameWrappingRenderer = frameWrappingRenderer;
+	}
+
 	/**
 	 * @param httpHeaderProcessor which should process HTTP headers
 	 */
@@ -77,6 +92,51 @@
 			ResultURIConverter uriConverter, CaptureSearchResults results)
 			throws ServletException, IOException, WaybackException {
 
+		// The URL of the page, for resolving in-page relative URLs: 
+    	URL url = null;
+		try {
+			url = new URL(result.getOriginalUrl());
+		} catch (MalformedURLException e1) {
+			// TODO: this shouldn't happen...
+			e1.printStackTrace();
+			throw new IOException(e1.getMessage());
+		}
+		// determine the character set used to encode the document bytes:
+		String charSet = charsetDetector.getCharset(resource, wbRequest);
+
+		ArchivalUrlContextResultURIConverterFactory fact = 
+			new ArchivalUrlContextResultURIConverterFactory(
+					(ArchivalUrlResultURIConverter) uriConverter);
+		// set up the context:
+		ReplayParseContext context = 
+			new ReplayParseContext(fact,url,result.getCaptureTimestamp());
+
+		if(!wbRequest.isFrameWrapperContext()) {
+			// in case this is an HTML page with FRAMEs, peek ahead an look:
+			// TODO: make ThreadLocal:
+			byte buffer[] = new byte[FRAMESET_SCAN_BUFFER_SIZE];
+
+			resource.mark(FRAMESET_SCAN_BUFFER_SIZE);
+			int amtRead = resource.read(buffer);
+			resource.reset();
+			
+			if(amtRead > 0) {
+				StringBuilder foo = new StringBuilder(new String(buffer,charSet));
+				int frameIdx = TagMagix.getEndOfFirstTag(foo, "FRAMESET");
+				if(frameIdx != -1) {
+					// insert flag so we don't add FRAMESET:
+					context.putData(FastArchivalUrlReplayParseEventHandler.FERRET_DONE_KEY,"");
+
+//					// top-level Frameset: Draw the frame wrapper thingy:
+//					frameWrappingRenderer.renderResource(httpRequest, 
+//							httpResponse, wbRequest, result, resource, 
+//							uriConverter, results);
+//					return;
+				}
+			}
+		}
+		
+		
 		// copy the HTTP response code:
 		HttpHeaderOperation.copyHTTPMessageHeader(resource, httpResponse);
 
@@ -90,31 +150,14 @@
 		JSPExecutor jspExec = new JSPExecutor(uriConverter, httpRequest, 
 				httpResponse, wbRequest, results, result, resource);
 		
-		// The URL of the page, for resolving in-page relative URLs: 
-    	URL url = null;
-		try {
-			url = new URL(result.getOriginalUrl());
-		} catch (MalformedURLException e1) {
-			// TODO: this shouldn't happen...
-			e1.printStackTrace();
-			throw new IOException(e1.getMessage());
-		}
 
 		// To make sure we get the length, we have to buffer it all up...
 		ByteArrayOutputStream baos = new ByteArrayOutputStream();
 
-		ArchivalUrlContextResultURIConverterFactory fact = 
-			new ArchivalUrlContextResultURIConverterFactory(
-					(ArchivalUrlResultURIConverter) uriConverter);
-		// set up the context:
-		ReplayParseContext context = 
-			new ReplayParseContext(fact,url,result.getCaptureTimestamp());
 		context.setOutputCharset(OUTPUT_CHARSET);
 		context.setOutputStream(baos);
 		context.setJspExec(jspExec);
 
-		// determine the character set used to encode the document bytes:
-		String charSet = charsetDetector.getCharset(resource, wbRequest);
 		
 		// and finally, parse, using the special lexer that knows how to
 		// handle javascript blocks containing unescaped HTML entities:


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.