[Archive-access-cvs] SF.net SVN: archive-access:[3485] trunk/archive-access/projects/wayback/ wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/ RobotExclusionFilter.java

[<bra...@us...>]

Revision: 3485
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3485&view=rev
Author:   bradtofel
Date:     2011-07-08 04:47:51 +0000 (Fri, 08 Jul 2011)

Log Message:
-----------
LOGGING: changed logging levels for most messages, added PerformanceLogger

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilter.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilter.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilter.java	2011-07-08 04:40:04 UTC (rev 3484)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilter.java	2011-07-08 04:47:51 UTC (rev 3485)
@@ -40,6 +40,7 @@
 import org.archive.wayback.resourceindex.filters.ExclusionFilter;
 import org.archive.wayback.util.ObjectFilter;
 import org.archive.wayback.util.url.UrlOperations;
+import org.archive.wayback.webapp.PerformanceLogger;
 
 /**
  * CaptureSearchResult Filter that uses a LiveWebCache to retrieve robots.txt
@@ -172,16 +173,20 @@
 				LOGGER.fine("ROBOT: Cached("+urlString+")");
 				rules = rulesCache.get(urlString);
 				if(!urlString.equals(firstUrlString)) {
-					LOGGER.info("Adding extra url("+firstUrlString+") for prev cached rules("+urlString+")");
+					LOGGER.fine("Adding extra url("+firstUrlString+") for prev cached rules("+urlString+")");
 					rulesCache.put(firstUrlString, rules);
 				}
 			} else {
 				try {
-					LOGGER.info("ROBOT: NotCached - Downloading("+urlString+")");
+					LOGGER.fine("ROBOT: NotCached - Downloading("+urlString+")");
 				
 					tmpRules = new RobotRules();
+					long start = System.currentTimeMillis();
 					Resource resource = webCache.getCachedResource(new URL(urlString),
 							maxCacheMS,true);
+					long elapsed = System.currentTimeMillis() - start;
+					PerformanceLogger.noteElapsed("RobotRequest", elapsed, urlString);
+
 					if(resource.getStatusCode() != 200) {
 						LOGGER.info("ROBOT: NotAvailable("+urlString+")");
 						throw new LiveDocumentNotAvailableException(urlString);
@@ -189,24 +194,24 @@
 					tmpRules.parse(resource);
 					rulesCache.put(firstUrlString,tmpRules);
 					rules = tmpRules;
-					LOGGER.info("ROBOT: Downloaded("+urlString+")");
+					LOGGER.fine("ROBOT: Downloaded("+urlString+")");
 
 				} catch (LiveDocumentNotAvailableException e) {
 					LOGGER.info("ROBOT: LiveDocumentNotAvailableException("+urlString+")");
 
 				} catch (MalformedURLException e) {
 //					e.printStackTrace();
-					LOGGER.info("ROBOT: MalformedURLException("+urlString+")");
+					LOGGER.warning("ROBOT: MalformedURLException("+urlString+")");
 					return null;
 				} catch (IOException e) {
 					LOGGER.warning("ROBOT: IOException("+urlString+"):"+e.getLocalizedMessage());
 					return null;
 				} catch (LiveWebCacheUnavailableException e) {
-					LOGGER.info("ROBOT: LiveWebCacheUnavailableException("+urlString+")");
+					LOGGER.severe("ROBOT: LiveWebCacheUnavailableException("+urlString+")");
 					filterGroup.setLiveWebGone();
 					return null;
 				} catch (LiveWebTimeoutException e) {
-					LOGGER.info("ROBOT: LiveDocumentTimedOutException("+urlString+")");
+					LOGGER.severe("ROBOT: LiveDocumentTimedOutException("+urlString+")");
 					filterGroup.setRobotTimedOut();
 					return null;
 				}
@@ -216,7 +221,7 @@
 			// special-case, allow empty rules if no longer available.
 			rulesCache.put(firstUrlString,emptyRules);
 			rules = emptyRules;
-			LOGGER.info("No rules available, using emptyRules for:" + firstUrlString);
+			LOGGER.fine("No rules available, using emptyRules for:" + firstUrlString);
 		}
 		return rules;
 	}
@@ -257,9 +262,9 @@
 					notifiedPassed = true;
 				}
 				filterResult = ObjectFilter.FILTER_INCLUDE;
-				LOGGER.fine("ROBOT: ALLOWED("+resultURL+")");
+				LOGGER.finer("ROBOT: ALLOWED("+resultURL+")");
 			} else {
-				LOGGER.info("ROBOT: BLOCKED("+resultURL+")");
+				LOGGER.fine("ROBOT: BLOCKED("+resultURL+")");
 			}
 		}
 		return filterResult;


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.