From: <bra...@us...> - 2007-07-20 01:24:28
|
Revision: 1850 http://archive-access.svn.sourceforge.net/archive-access/?rev=1850&view=rev Author: bradtofel Date: 2007-07-19 18:24:30 -0700 (Thu, 19 Jul 2007) Log Message: ----------- REFACTOR: changed base class from WaybackServlet to org.archive.wayback.webapp.ServletRequestContext, and added getter/setter for Spring configuration Modified Paths: -------------- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionServlet.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionServlet.java Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionServlet.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionServlet.java 2007-07-20 01:24:06 UTC (rev 1849) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionServlet.java 2007-07-20 01:24:30 UTC (rev 1850) @@ -37,9 +37,8 @@ import org.apache.commons.httpclient.URIException; import org.archive.wayback.core.Timestamp; -import org.archive.wayback.core.WaybackServlet; -import org.archive.wayback.exception.ConfigurationException; import org.archive.wayback.surt.SURTTokenizer; +import org.archive.wayback.webapp.ServletRequestContext; import com.sleepycat.je.DatabaseException; @@ -55,9 +54,8 @@ * @author brad * @version $Date$, $Revision$ */ -public class AdministrativeExclusionServlet extends WaybackServlet { +public class AdministrativeExclusionServlet extends ServletRequestContext { - private static String ADMIN_EXCLUSION_AUTH_CLASSNAME = "exclusionauthority"; private static final String DEFAULT_USER_AGENT = "ia_archiver"; private static final String HTML_BR = "<br></br>"; private static final long serialVersionUID = 1L; @@ -102,25 +100,8 @@ private static final int RULE_STATUS_INACTIVE = 1; private static final int RULE_STATUS_DELETE = 2; + private AdministrativeExclusionAuthority exclusionAuthority = null; - /** - * possibly initializes and returns the queryRenderer - * - * @return Returns the RoboCache. - * @throws ServletException - */ - public AdministrativeExclusionAuthority getExclusionAuthority() - throws ServletException { - try { - // TODO: verify type - return (AdministrativeExclusionAuthority) wayback - .getCachedInstance(ADMIN_EXCLUSION_AUTH_CLASSNAME); - } catch (ConfigurationException e) { - e.printStackTrace(); - throw new ServletException(e); - } - } - private void showPage(HttpServletResponse response, StringBuilder page) throws IOException { response.setContentType("text/html"); @@ -131,14 +112,15 @@ return e.getMessage(); } - public void doGet(HttpServletRequest httpRequest, - HttpServletResponse httpResponse) throws IOException, - ServletException { - AdministrativeExclusionAuthority exclAuth = getExclusionAuthority(); + public boolean handleRequest(HttpServletRequest httpRequest, + HttpServletResponse httpResponse) throws ServletException, + IOException { + AdministrativeExclusionAuthority exclAuth = exclusionAuthority; StringBuilder page = new StringBuilder(1024); page.append("<html><head><title>Wayback: Administrative Exclusions</title></head><body>"); - Map queryArgs = httpRequest.getParameterMap(); + @SuppressWarnings("unchecked") + Map<String,String[]> queryArgs = httpRequest.getParameterMap(); String operation = getMapParam(queryArgs,FORM_OPERATION); if(operation == null) { page.append(makeCheckForm(queryArgs)); @@ -174,25 +156,10 @@ } page.append(makeQueryForm(queryArgs)); page.append(makeCreateForm(queryArgs)); + + } else if((operation.equals(MODIFY_FORM_OPERATION) + || operation.equals(DELETE_FORM_OPERATION))) { - } else { - page.append("GET request cannot change database"); - } - page.append("</body></html>"); - showPage(httpResponse,page); - } - public void doPost(HttpServletRequest httpRequest, - HttpServletResponse httpResponse) throws IOException, - ServletException { - AdministrativeExclusionAuthority exclAuth = getExclusionAuthority(); - Map queryArgs = httpRequest.getParameterMap(); - String operation = getMapParam(queryArgs,FORM_OPERATION); - StringBuilder page = new StringBuilder(1024); - page.append("<html><head><title>Wayback: Administrative Exclusions</title></head><body>"); - page.append(makeCheckForm(queryArgs)); - page.append(makeQueryForm(queryArgs)); - page.append(makeCreateForm(queryArgs)); - if(operation != null && (operation.equals(MODIFY_FORM_OPERATION) || operation.equals(DELETE_FORM_OPERATION))) { try { handleRuleCreate(exclAuth,queryArgs); page.append("OK - added rule"); @@ -206,12 +173,15 @@ e.printStackTrace(); page.append(formatException(e)); } + + } else { + page.append("Unknown operation"); } page.append("</body></html>"); showPage(httpResponse,page); + return true; } - - + // HTML GENERATION METHODS: private String htmlEncode(String orig) { @@ -221,7 +191,8 @@ return orig; } - private String makeFormTextInput(String label, String name, Map queryArgs, + private String makeFormTextInput(String label, String name, + Map<String,String[]> queryArgs, String suffix) { String value = ""; if(queryArgs != null) { @@ -231,7 +202,7 @@ value + "\"></input>" + suffix; } private String makeFormTextAreaInput(String label, String name, - Map queryArgs, String suffix) { + Map<String,String[]> queryArgs, String suffix) { String value = ""; if(queryArgs != null) { value = htmlEncode(getMapParamOrEmpty(queryArgs,name)); @@ -241,7 +212,7 @@ } private String makeFormCheckInput(String label, String name, String value, - Map queryArgs, String suffix) { + Map<String,String[]> queryArgs, String suffix) { String curValue = getMapParam(queryArgs,name); String checked = ""; if(curValue != null && curValue.equals(value)) { @@ -274,14 +245,14 @@ // CHECK FORM AND HANDLING - private String makeCheckForm(Map queryArgs) { + private String makeCheckForm(Map<String,String[]> queryArgs) { String content = makeFormTextInput("Url",CHECK_FORM_URL,queryArgs," ") + makeFormTextInput("Timestamp",CHECK_FORM_TIMESTAMP,queryArgs," "); return makeHeaderForm("Test Exclusion","GET",CHECK_FORM_OPERATION,content); } private String handleRuleCheck(AdministrativeExclusionAuthority excl, - Map queryArgs) throws Exception { + Map<String,String[]> queryArgs) throws Exception { String url = getRequiredMapParam(queryArgs,CHECK_FORM_URL); String timestamp = getRequiredMapParam(queryArgs,CHECK_FORM_TIMESTAMP); String userAgent = DEFAULT_USER_AGENT; @@ -292,7 +263,7 @@ // RULE QUERY FORM AND HANDLING - private String makeQueryForm(Map queryArgs) { + private String makeQueryForm(Map<String,String[]> queryArgs) { StringBuilder content = new StringBuilder(1024); content.append(makeFormTextInput("Url",QUERY_FORM_URL,queryArgs,"")); content.append(makeFormCheckInput("all",QUERY_FORM_ALL,QUERY_FORM_ALL_VALUE,queryArgs,"")); @@ -300,12 +271,13 @@ } private String handleRuleQuery(AdministrativeExclusionAuthority excl, - Map queryArgs) throws ParseException, URIException, DatabaseException { + Map<String,String[]> queryArgs) throws ParseException, URIException, + DatabaseException { String url = getRequiredMapParam(queryArgs,QUERY_FORM_URL); String all = getMapParam(queryArgs,QUERY_FORM_ALL); boolean showAll = (all != null && all.equals(QUERY_FORM_ALL_VALUE)); String surt = SURTTokenizer.prefixKey(url); - ArrayList matching = excl.matchRules(surt); + ArrayList<AdministrativeExclusionRules> matching = excl.matchRules(surt); StringBuilder matchHTML = new StringBuilder(); rulesToTable(matchHTML,matching,url,showAll); return matchHTML.toString(); @@ -339,7 +311,7 @@ return makeForm("POST", DELETE_FORM_OPERATION, sb.toString()); } - private String makeCreateForm(Map queryArgs) { + private String makeCreateForm(Map<String,String[]> queryArgs) { StringBuilder sb = new StringBuilder(1024); sb.append(makeFormTextInput("Url",MODIFY_FORM_URL,queryArgs,"")); sb.append(makeFormCheckInput("(prefix)",MODIFY_FORM_PREFIX, @@ -369,7 +341,7 @@ } private void handleRuleCreate(AdministrativeExclusionAuthority auth, - Map queryMap) throws ParseException, URIException, DatabaseException { + Map<String,String[]> queryMap) throws ParseException, URIException, DatabaseException { AdministrativeExclusionRule rule = new AdministrativeExclusionRule(); @@ -443,7 +415,8 @@ String key = rule.key(); boolean deleted = false; - Iterator itr = rules.getRules().iterator(); + Iterator<AdministrativeExclusionRule> itr = + rules.getRules().iterator(); while(itr.hasNext()) { AdministrativeExclusionRule daRule = (AdministrativeExclusionRule) itr.next(); @@ -519,19 +492,19 @@ page.append("</tr>\n"); } - private void rulesToTable(StringBuilder page,ArrayList matching, - String url, boolean showAll) { + private void rulesToTable(StringBuilder page, + ArrayList<AdministrativeExclusionRules> matching, String url, + boolean showAll) { if(matching.size() > 0) { page.append("<table border=1>"); ruleHeaderRow(page); - Iterator itr = matching.iterator(); + Iterator<AdministrativeExclusionRules> itr = matching.iterator(); while(itr.hasNext()) { - AdministrativeExclusionRules rules = - (AdministrativeExclusionRules) itr.next(); + AdministrativeExclusionRules rules = itr.next(); String surtPrefix = rules.getSurtPrefix(); - Iterator ruleItr = null; + Iterator<AdministrativeExclusionRule> ruleItr = null; if(showAll) { ruleItr = rules.getRules().iterator(); } else { @@ -549,6 +522,4 @@ page.append("<p>No records match("+url+")</p>\n"); } } - - } Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionServlet.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionServlet.java 2007-07-20 01:24:06 UTC (rev 1849) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionServlet.java 2007-07-20 01:24:30 UTC (rev 1850) @@ -27,13 +27,11 @@ import java.io.IOException; import java.io.OutputStream; import java.util.Map; -//import java.util.logging.Logger; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.archive.wayback.core.WaybackServlet; -import org.archive.wayback.exception.ConfigurationException; +import org.archive.wayback.webapp.ServletRequestContext; /** * @@ -41,15 +39,9 @@ * @author brad * @version $Date$, $Revision$ */ -public class ExclusionServlet extends WaybackServlet { +public class ExclusionServlet extends ServletRequestContext { private static final long serialVersionUID = 1L; -// private static final Logger LOGGER = Logger -// .getLogger(ExclusionServlet.class.getName()); - - private final static String EXCLUSION_AUTHORITY_CLASSNAME = - "exclusionauthority"; - private static final String OPERATION_ARGUMENT = "operation"; private static final String URL_ARGUMENT = "url"; @@ -60,30 +52,17 @@ private static final String OPERATION_CHECK = "check"; + private ExclusionAuthority exclusionAuthority = null; // private static final String OPERATION_PURGE = "purge"; - /** - * possibly initializes and returns the ExclusionAuthority - * - * @return Returns the RoboCache. - * @throws ServletException - */ - public ExclusionAuthority getExclusionAuthority() throws ServletException { - try { - return (ExclusionAuthority) wayback - .getCachedInstance(EXCLUSION_AUTHORITY_CLASSNAME); - } catch (ConfigurationException e) { - e.printStackTrace(); - throw new ServletException(e); - } - } - public void doGet(HttpServletRequest httpRequest, + public boolean handleRequest(HttpServletRequest httpRequest, HttpServletResponse httpResponse) throws IOException, ServletException { - ExclusionAuthority exclAuth = getExclusionAuthority(); + ExclusionAuthority exclAuth = exclusionAuthority; - Map queryArgs = httpRequest.getParameterMap(); + @SuppressWarnings("unchecked") + Map<String,String[]> queryArgs = httpRequest.getParameterMap(); String url = getMapParam(queryArgs, URL_ARGUMENT); String operation = getMapParam(queryArgs, OPERATION_ARGUMENT); String userAgent = getMapParam(queryArgs, USER_AGENT_ARGUMENT); @@ -139,5 +118,20 @@ httpResponse.setContentType(eclResponse.getContentType()); OutputStream os = httpResponse.getOutputStream(); eclResponse.writeResponse(os); + return true; } + + /** + * @param exclusionAuthority the exclusionAuthority to set + */ + public void setExclusionAuthority(ExclusionAuthority exclusionAuthority) { + this.exclusionAuthority = exclusionAuthority; + } + + /** + * @return the exclusionAuthority + */ + public ExclusionAuthority getExclusionAuthority() { + return exclusionAuthority; + } } This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <bra...@us...> - 2007-07-25 00:43:21
|
Revision: 1880 http://archive-access.svn.sourceforge.net/archive-access/?rev=1880&view=rev Author: bradtofel Date: 2007-07-24 17:43:20 -0700 (Tue, 24 Jul 2007) Log Message: ----------- REFACTOR: removed all references to PropertyConfigurable interface Modified Paths: -------------- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionAuthority.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionAuthority.java Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionAuthority.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionAuthority.java 2007-07-25 00:41:50 UTC (rev 1879) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionAuthority.java 2007-07-25 00:43:20 UTC (rev 1880) @@ -24,16 +24,12 @@ */ package org.archive.wayback.accesscontrol; -import java.io.File; import java.util.ArrayList; -import java.util.Properties; import org.apache.commons.httpclient.URIException; import org.archive.wayback.bdb.BDBRecord; import org.archive.wayback.bdb.BDBRecordSet; import org.archive.wayback.bdb.BDBRecordIterator; -import org.archive.wayback.core.PropertyConfiguration; -import org.archive.wayback.exception.ConfigurationException; import org.archive.wayback.surt.SURTTokenizer; import com.sleepycat.je.DatabaseException; @@ -47,15 +43,13 @@ */ public class AdministrativeExclusionAuthority implements ExclusionAuthority { - + // TODO: read from ResounceBundle private static String ADMIN_NO_ROBOTS_MSG = "Administrative Robots Ignore:"; private static String ADMIN_INCLUDE_MSG = "Administrative Include:"; private static String ADMIN_EXCLUDE_MSG = "Administrative Exclude:"; - private static String DB_NAME = "adminexclusion.dbname"; - private static String INDEX_PATH = "adminexclusion.dbpath"; // RoboCache roboCache; - BDBRecordSet db; + private BDBRecordSet db = null; /* (non-Javadoc) * @see org.archive.wayback.accesscontrol.ExclusionAuthority#checkExclusion(java.lang.String, java.lang.String, java.lang.String) @@ -133,27 +127,7 @@ rules.loadRules(encodedRules); return rules; } - /* (non-Javadoc) - * @see org.archive.wayback.PropertyConfigurable#init(java.util.Properties) - */ - public void init(Properties p) throws ConfigurationException { - PropertyConfiguration pc = new PropertyConfiguration(p); - File dbDir = pc.getDir(INDEX_PATH,true); - String dbName = pc.getString(DB_NAME); - try { - db = new BDBRecordSet(); - db.initializeDB(dbDir.getAbsolutePath(),dbName); - - } catch (DatabaseException e) { - e.printStackTrace(); - throw new ConfigurationException(e.getMessage()); - } - -// roboCache = new RoboCache(); -// roboCache.init(p); - } - private AdministrativeExclusionRule getRuleFor(final String surtPrefix, final String dateStr) throws DatabaseException { AdministrativeExclusionRules rules = new AdministrativeExclusionRules(surtPrefix); @@ -178,4 +152,18 @@ rules.addRule(rule); db.put(surtPrefix,rules.encodeRules()); } + + /** + * @return the db + */ + public BDBRecordSet getDb() { + return db; + } + + /** + * @param db the db to set + */ + public void setDb(BDBRecordSet db) { + this.db = db; + } } Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionAuthority.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionAuthority.java 2007-07-25 00:41:50 UTC (rev 1879) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionAuthority.java 2007-07-25 00:43:20 UTC (rev 1880) @@ -24,15 +24,13 @@ */ package org.archive.wayback.accesscontrol; -import org.archive.wayback.PropertyConfigurable; - /** * * * @author brad * @version $Date$, $Revision$ */ -public interface ExclusionAuthority extends PropertyConfigurable{ +public interface ExclusionAuthority { /** * determines whether userAgent can view urlString for captureDate, * encapsulating the response in a returned ExclusionResponse object This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <bra...@us...> - 2007-07-25 00:44:45
|
Revision: 1881 http://archive-access.svn.sourceforge.net/archive-access/?rev=1881&view=rev Author: bradtofel Date: 2007-07-24 17:44:45 -0700 (Tue, 24 Jul 2007) Log Message: ----------- REFACTOR: removed all references to PropertyConfigurable interface TWEAK: type safety Modified Paths: -------------- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/remote/RemoteExclusionFilterFactory.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilterFactory.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterFactory.java Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/remote/RemoteExclusionFilterFactory.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/remote/RemoteExclusionFilterFactory.java 2007-07-25 00:43:20 UTC (rev 1880) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/remote/RemoteExclusionFilterFactory.java 2007-07-25 00:44:45 UTC (rev 1881) @@ -24,12 +24,9 @@ */ package org.archive.wayback.accesscontrol.remote; -import java.util.Properties; - -import org.archive.wayback.core.PropertyConfiguration; -import org.archive.wayback.exception.ConfigurationException; +import org.archive.wayback.core.SearchResult; import org.archive.wayback.resourceindex.ExclusionFilterFactory; -import org.archive.wayback.resourceindex.SearchResultFilter; +import org.archive.wayback.util.ObjectFilter; /** * @@ -39,36 +36,43 @@ */ public class RemoteExclusionFilterFactory implements ExclusionFilterFactory { + private String exclusionUrlPrefix = null; + + private String exclusionUserAgent = null; + + /* (non-Javadoc) + * @see org.archive.wayback.resourceindex.ExclusionFilterFactory#get() + */ + public ObjectFilter<SearchResult> get() { + return new RemoteExclusionFilter(exclusionUrlPrefix, exclusionUserAgent); + } + /** - * configuration name for URL prefix to access exclusion service + * @return the exclusionUrlPrefix */ - private final static String EXCLUSION_PREFIX = "resourceindex.exclusionurl"; + public String getExclusionUrlPrefix() { + return exclusionUrlPrefix; + } /** - * configuration name for User Agent to send to exclusion service + * @param exclusionUrlPrefix the exclusionUrlPrefix to set */ - private final static String EXCLUSION_UA = "resourceindex.exclusionua"; + public void setExclusionUrlPrefix(String exclusionUrlPrefix) { + this.exclusionUrlPrefix = exclusionUrlPrefix; + } - - private String exclusionUrlPrefix = null; - - private String exclusionUserAgent = null; - - - /* (non-Javadoc) - * @see org.archive.wayback.PropertyConfigurable#init(java.util.Properties) + /** + * @return the exclusionUserAgent */ - public void init(Properties p) throws ConfigurationException { - PropertyConfiguration pc = new PropertyConfiguration(p); - exclusionUrlPrefix = pc.getString(EXCLUSION_PREFIX); - exclusionUserAgent = pc.getString(EXCLUSION_UA); + public String getExclusionUserAgent() { + return exclusionUserAgent; } - /* (non-Javadoc) - * @see org.archive.wayback.resourceindex.ExclusionFilterFactory#get() + /** + * @param exclusionUserAgent the exclusionUserAgent to set */ - public SearchResultFilter get() { - return new RemoteExclusionFilter(exclusionUrlPrefix, exclusionUserAgent); + public void setExclusionUserAgent(String exclusionUserAgent) { + this.exclusionUserAgent = exclusionUserAgent; } Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilterFactory.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilterFactory.java 2007-07-25 00:43:20 UTC (rev 1880) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilterFactory.java 2007-07-25 00:44:45 UTC (rev 1881) @@ -24,13 +24,10 @@ */ package org.archive.wayback.accesscontrol.robotstxt; -import java.util.Properties; - -import org.archive.wayback.core.PropertyConfiguration; -import org.archive.wayback.exception.ConfigurationException; +import org.archive.wayback.core.SearchResult; import org.archive.wayback.liveweb.LiveWebCache; import org.archive.wayback.resourceindex.ExclusionFilterFactory; -import org.archive.wayback.resourceindex.SearchResultFilter; +import org.archive.wayback.util.ObjectFilter; /** * @@ -40,29 +37,56 @@ */ public class RobotExclusionFilterFactory implements ExclusionFilterFactory { - private final static String ROBOT_USER_AGENT = "robotexclusion.useragent"; - private final static String ROBOT_CACHE_AGE = "robotexclusion.cacheagems"; - - private LiveWebCache webCache = null; private String userAgent = null; private long maxCacheMS = 0; /* (non-Javadoc) - * @see org.archive.wayback.PropertyConfigurable#init(java.util.Properties) + * @see org.archive.wayback.resourceindex.ExclusionFilterFactory#get() */ - public void init(Properties p) throws ConfigurationException { - PropertyConfiguration pc = new PropertyConfiguration(p); - userAgent = pc.getString(ROBOT_USER_AGENT); - maxCacheMS = pc.getLong(ROBOT_CACHE_AGE); - webCache = new LiveWebCache(); - webCache.init(p); + public ObjectFilter<SearchResult> get() { + return new RobotExclusionFilter(webCache,userAgent,maxCacheMS); } - /* (non-Javadoc) - * @see org.archive.wayback.resourceindex.ExclusionFilterFactory#get() + /** + * @return the webCache */ - public SearchResultFilter get() { - return new RobotExclusionFilter(webCache,userAgent,maxCacheMS); + public LiveWebCache getWebCache() { + return webCache; } + + /** + * @param webCache the webCache to set + */ + public void setWebCache(LiveWebCache webCache) { + this.webCache = webCache; + } + + /** + * @return the userAgent + */ + public String getUserAgent() { + return userAgent; + } + + /** + * @param userAgent the userAgent to set + */ + public void setUserAgent(String userAgent) { + this.userAgent = userAgent; + } + + /** + * @return the maxCacheMS + */ + public long getMaxCacheMS() { + return maxCacheMS; + } + + /** + * @param maxCacheMS the maxCacheMS to set + */ + public void setMaxCacheMS(long maxCacheMS) { + this.maxCacheMS = maxCacheMS; + } } Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterFactory.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterFactory.java 2007-07-25 00:43:20 UTC (rev 1880) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterFactory.java 2007-07-25 00:44:45 UTC (rev 1881) @@ -28,15 +28,13 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.Properties; import java.util.logging.Logger; -import org.archive.wayback.core.PropertyConfiguration; -import org.archive.wayback.exception.ConfigurationException; +import org.archive.wayback.core.SearchResult; import org.archive.wayback.resourceindex.ExclusionFilterFactory; -import org.archive.wayback.resourceindex.SearchResultFilter; import org.archive.wayback.surt.SURTTokenizer; import org.archive.wayback.util.CloseableIterator; +import org.archive.wayback.util.ObjectFilter; import org.archive.wayback.util.flatfile.FlatFile; /** @@ -49,12 +47,9 @@ private static final Logger LOGGER = Logger.getLogger(StaticMapExclusionFilterFactory.class.getName()); - - private final static String EXCLUSION_PATH = - "resourceindex.exclusionpath"; private final static int checkInterval = 10; - Map<String,Object> currentMap = null; - File file = null; + private Map<String,Object> currentMap = null; + private File file = null; long lastUpdated = 0; /** * Thread object of update thread -- also is flag indicating if the thread @@ -62,23 +57,15 @@ */ private static Thread updateThread = null; - - /* (non-Javadoc) - * @see org.archive.wayback.PropertyConfigurable#init(java.util.Properties) + /** + * load exclusion file and startup polling thread to check for updates + * @throws IOException */ - public void init(Properties p) throws ConfigurationException { - PropertyConfiguration pc = new PropertyConfiguration(p); - file = pc.getFile(EXCLUSION_PATH); - try { - reloadFile(); - } catch (IOException e) { - e.printStackTrace(); - throw new ConfigurationException(e.getLocalizedMessage()); - } - LOGGER.info("starting CachedMapExclusion with file " + - file.getAbsolutePath()); - startup(); + public void init() throws IOException { + reloadFile(); + startUpdateThread(); } + protected void reloadFile() throws IOException { long currentMod = file.lastModified(); if(currentMod == lastUpdated) { @@ -98,32 +85,27 @@ protected Map<String,Object> loadFile(String path) throws IOException { Map<String, Object> newMap = new HashMap<String, Object>(); FlatFile ff = new FlatFile(path); - CloseableIterator itr = (CloseableIterator) ff.getSequentialIterator(); + CloseableIterator<String> itr = ff.getSequentialIterator(); while(itr.hasNext()) { String line = (String) itr.next(); String surt = line.startsWith("(") ? line : SURTTokenizer.prefixKey(line); newMap.put(surt, null); } + itr.close(); return newMap; } /** * @return SearchResultFilter */ - public SearchResultFilter get() { + public ObjectFilter<SearchResult> get() { if(currentMap == null) { return null; } return new StaticMapExclusionFilter(currentMap); } - private void startup() throws ConfigurationException { - if (updateThread == null) { - startUpdateThread(); - } - } - private synchronized void startUpdateThread() { if (updateThread != null) { return; This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <bra...@us...> - 2007-07-25 00:45:34
|
Revision: 1882 http://archive-access.svn.sourceforge.net/archive-access/?rev=1882&view=rev Author: bradtofel Date: 2007-07-24 17:45:21 -0700 (Tue, 24 Jul 2007) Log Message: ----------- REFACTOR: now implement ObjectFilter with strong typing, instead of inheriting from SearchResultFilter Modified Paths: -------------- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/remote/RemoteExclusionFilter.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilter.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilter.java Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/remote/RemoteExclusionFilter.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/remote/RemoteExclusionFilter.java 2007-07-25 00:44:45 UTC (rev 1881) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/remote/RemoteExclusionFilter.java 2007-07-25 00:45:21 UTC (rev 1882) @@ -34,7 +34,7 @@ import org.archive.wayback.WaybackConstants; import org.archive.wayback.core.SearchResult; -import org.archive.wayback.resourceindex.SearchResultFilter; +import org.archive.wayback.util.ObjectFilter; /** * SearchResultFilter which uses remote access control/exclusion service to @@ -43,7 +43,7 @@ * @author brad * @version $Date: 2006-10-17 15:21:15 -0700 (Tue, 17 Oct 2006) $, $Revision: 1276 $ */ -public class RemoteExclusionFilter extends SearchResultFilter { +public class RemoteExclusionFilter implements ObjectFilter<SearchResult> { private static final Logger LOGGER = Logger.getLogger(RemoteExclusionFilter.class .getName()); @@ -146,7 +146,7 @@ /* (non-Javadoc) * @see org.archive.wayback.resourceindex.SearchResultFilter#filterSearchResult(org.archive.wayback.core.SearchResult) */ - public int filterSearchResult(SearchResult r) { + public int filterObject(SearchResult r) { String captureDate = r.get(WaybackConstants.RESULT_CAPTURE_DATE); String url = r.get(WaybackConstants.RESULT_URL); return isBlocked(url,captureDate) ? FILTER_EXCLUDE : FILTER_INCLUDE; Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilter.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilter.java 2007-07-25 00:44:45 UTC (rev 1881) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilter.java 2007-07-25 00:45:21 UTC (rev 1882) @@ -40,7 +40,7 @@ import org.archive.wayback.core.SearchResult; import org.archive.wayback.exception.LiveDocumentNotAvailableException; import org.archive.wayback.liveweb.LiveWebCache; -import org.archive.wayback.resourceindex.SearchResultFilter; +import org.archive.wayback.util.ObjectFilter; /** * SearchResultFilter that uses a LiveWebCache to retrieve robots.txt documents @@ -56,7 +56,7 @@ * @author brad * @version $Date$, $Revision$ */ -public class RobotExclusionFilter extends SearchResultFilter { +public class RobotExclusionFilter implements ObjectFilter<SearchResult> { private final static String HTTP_PREFIX = "http://"; private final static String ROBOT_SUFFIX = "/robots.txt"; @@ -175,9 +175,9 @@ /* (non-Javadoc) * @see org.archive.wayback.resourceindex.SearchResultFilter#filterSearchResult(org.archive.wayback.core.SearchResult) */ - public int filterSearchResult(SearchResult r) { + public int filterObject(SearchResult r) { - int filterResult = SearchResultFilter.FILTER_EXCLUDE; + int filterResult = ObjectFilter.FILTER_EXCLUDE; RobotRules rules = getRules(r); if(rules != null) { String resultURL = r.get(WaybackConstants.RESULT_URL); @@ -185,7 +185,7 @@ try { url = new URL(ArchiveUtils.addImpliedHttpIfNecessary(resultURL)); if(!rules.blocksPathForUA(url.getPath(), userAgent)) { - filterResult = SearchResultFilter.FILTER_INCLUDE; + filterResult = ObjectFilter.FILTER_INCLUDE; } } catch (MalformedURLException e) { e.printStackTrace(); Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilter.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilter.java 2007-07-25 00:44:45 UTC (rev 1881) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilter.java 2007-07-25 00:45:21 UTC (rev 1882) @@ -29,8 +29,8 @@ import org.apache.commons.httpclient.URIException; import org.archive.wayback.WaybackConstants; import org.archive.wayback.core.SearchResult; -import org.archive.wayback.resourceindex.SearchResultFilter; import org.archive.wayback.surt.SURTTokenizer; +import org.archive.wayback.util.ObjectFilter; /** * @@ -38,7 +38,7 @@ * @author brad * @version $Date$, $Revision$ */ -public class StaticMapExclusionFilter extends SearchResultFilter { +public class StaticMapExclusionFilter implements ObjectFilter<SearchResult> { private String lastChecked = null; private boolean lastCheckedExcluded = false; @@ -71,19 +71,19 @@ /* (non-Javadoc) * @see org.archive.wayback.resourceindex.SearchResultFilter#filterSearchResult(org.archive.wayback.core.SearchResult) */ - public int filterSearchResult(SearchResult r) { + public int filterObject(SearchResult r) { String url = r.get(WaybackConstants.RESULT_URL); if(lastChecked != null) { if(lastChecked.equals(url)) { return lastCheckedExcluded ? - SearchResultFilter.FILTER_EXCLUDE : - SearchResultFilter.FILTER_INCLUDE; + ObjectFilter.FILTER_EXCLUDE : + ObjectFilter.FILTER_INCLUDE; } } lastChecked = url; lastCheckedExcluded = isExcluded(url); return lastCheckedExcluded ? - SearchResultFilter.FILTER_EXCLUDE : - SearchResultFilter.FILTER_INCLUDE; + ObjectFilter.FILTER_EXCLUDE : + ObjectFilter.FILTER_INCLUDE; } } This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <bra...@us...> - 2007-07-26 21:42:34
|
Revision: 1893 http://archive-access.svn.sourceforge.net/archive-access/?rev=1893&view=rev Author: bradtofel Date: 2007-07-26 14:42:35 -0700 (Thu, 26 Jul 2007) Log Message: ----------- REFACTOR: moved org.archive.wayback.accesscontrol.* classes to org.archive.wayback.accesscontrol.ui. Most of this stuff is not performant enough for production use, and probably needs a nearly complete overhaul. Added Paths: ----------- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/AdministrativeExclusionAuthority.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/AdministrativeExclusionRule.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/AdministrativeExclusionRules.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/AdministrativeExclusionServlet.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/ExclusionAuthority.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/ExclusionResponse.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/ExclusionServlet.java Removed Paths: ------------- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionAuthority.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionRule.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionRules.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionServlet.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionAuthority.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionResponse.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionServlet.java Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionAuthority.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionAuthority.java 2007-07-26 21:31:53 UTC (rev 1892) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionAuthority.java 2007-07-26 21:42:35 UTC (rev 1893) @@ -1,169 +0,0 @@ -/* AdministrativeExclusionAuthority - * - * $Id$ - * - * Created on 2:47:39 PM May 10, 2006. - * - * Copyright (C) 2006 Internet Archive. - * - * This file is part of wayback. - * - * wayback is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * wayback is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with wayback; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ -package org.archive.wayback.accesscontrol; - -import java.util.ArrayList; - -import org.apache.commons.httpclient.URIException; -import org.archive.wayback.bdb.BDBRecord; -import org.archive.wayback.bdb.BDBRecordSet; -import org.archive.wayback.bdb.BDBRecordIterator; -import org.archive.wayback.surt.SURTTokenizer; - -import com.sleepycat.je.DatabaseException; - - -/** - * - * - * @author brad - * @version $Date$, $Revision$ - */ -public class AdministrativeExclusionAuthority implements ExclusionAuthority { - - // TODO: read from ResounceBundle - private static String ADMIN_NO_ROBOTS_MSG = "Administrative Robots Ignore:"; - private static String ADMIN_INCLUDE_MSG = "Administrative Include:"; - private static String ADMIN_EXCLUDE_MSG = "Administrative Exclude:"; - -// RoboCache roboCache; - private BDBRecordSet db = null; - - /* (non-Javadoc) - * @see org.archive.wayback.accesscontrol.ExclusionAuthority#checkExclusion(java.lang.String, java.lang.String, java.lang.String) - */ - public ExclusionResponse checkExclusion(String userAgent, String urlString, - String captureDate) throws Exception { - SURTTokenizer tokenizer; - try { - tokenizer = new SURTTokenizer(urlString); - } catch (URIException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - throw new Exception(e); - } - - while(true) { - String surtPrefix = tokenizer.nextSearch(); - if(surtPrefix == null) { - break; - } - AdministrativeExclusionRule rule = getRuleFor(surtPrefix,captureDate); - if(rule != null) { - if(rule.isRobots()) { - break; - } else if(rule.isNoRobots()) { - return new ExclusionResponse("-", - ExclusionResponse.EXLCUSION_AUTHORITATIVE, - ExclusionResponse.EXLCUSION_AUTHORIZED, - ADMIN_NO_ROBOTS_MSG + rule.getWhy()); - } else if(rule.isInclude()) { - return new ExclusionResponse("-", - ExclusionResponse.EXLCUSION_AUTHORITATIVE, - ExclusionResponse.EXLCUSION_AUTHORIZED, - ADMIN_INCLUDE_MSG + rule.getWhy()); - } else if(rule.isExclude()) { - return new ExclusionResponse("-", - ExclusionResponse.EXLCUSION_AUTHORITATIVE, - ExclusionResponse.EXLCUSION_NOT_AUTHORIZED, - ADMIN_EXCLUDE_MSG + rule.getWhy()); - } else { - // whoops.. how'd this happen.. just ignore it. - } - } - } - - // we only get here when we are suppose to return the value from the - // current robots.txt document: -// return roboCache.checkExclusion(userAgent,urlString,captureDate); - return null; - } - - /** - * @param surt - * @return String representation of rules - * @throws DatabaseException - */ - public ArrayList<AdministrativeExclusionRules> matchRules(String surt) throws DatabaseException { - BDBRecordIterator itr = db.recordIterator(surt); - ArrayList<AdministrativeExclusionRules> matching = - new ArrayList<AdministrativeExclusionRules>(); - while(itr.hasNext()) { - BDBRecord record = (BDBRecord) itr.next(); - AdministrativeExclusionRules rules = recordToRules(record); - if(rules.getSurtPrefix().startsWith(surt)) { - matching.add(rules); - } - } - return matching; - } - - private AdministrativeExclusionRules recordToRules(BDBRecord record) { - String surtPrefix = new String(record.getKey().getData()); - String encodedRules = new String(record.getValue().getData()); - AdministrativeExclusionRules rules = new AdministrativeExclusionRules(surtPrefix); - rules.loadRules(encodedRules); - return rules; - } - - private AdministrativeExclusionRule getRuleFor(final String surtPrefix, - final String dateStr) throws DatabaseException { - AdministrativeExclusionRules rules = new AdministrativeExclusionRules(surtPrefix); - String encoded = (String) db.get(surtPrefix); - if(encoded != null) { - rules.loadRules(encoded); - } - return rules.getApplicableRule(dateStr); - } - - /** - * @param surtPrefix - * @param rule - * @throws DatabaseException - */ - public void addRuleFor(final String surtPrefix, AdministrativeExclusionRule rule) throws DatabaseException { - AdministrativeExclusionRules rules = new AdministrativeExclusionRules(surtPrefix); - String encoded = (String) db.get(surtPrefix); - if(encoded != null) { - rules.loadRules(encoded); - } - rules.addRule(rule); - db.put(surtPrefix,rules.encodeRules()); - } - - /** - * @return the db - */ - public BDBRecordSet getDb() { - return db; - } - - /** - * @param db the db to set - */ - public void setDb(BDBRecordSet db) { - this.db = db; - } -} Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionRule.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionRule.java 2007-07-26 21:31:53 UTC (rev 1892) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionRule.java 2007-07-26 21:42:35 UTC (rev 1893) @@ -1,332 +0,0 @@ -/* AdministrativeExclusionRuling - * - * $Id$ - * - * Created on 11:20:22 AM May 11, 2006. - * - * Copyright (C) 2006 Internet Archive. - * - * This file is part of wayback. - * - * wayback is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * wayback is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with wayback; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ -package org.archive.wayback.accesscontrol; - -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.net.URLEncoder; -import java.text.ParseException; - -import org.archive.wayback.core.Timestamp; - -/** - * - * - * @author brad - * @version $Date$, $Revision$ - */ -public class AdministrativeExclusionRule { - -// -// VALUE: <RULE>[,VALUE] -// RULE: <TS-START>-<TS-END>:<TYPE>:<MOD>:<WHO>:<WHEN>:<COMMENT> -// TS-START: [0-9]{0,14} -// TS-END: [0-9]{0,14} -// TYPE: ROBOTS|NOROBOTS|EXCLUDE|INCLUDE -// MOD: ADD|DELETE -// WHO: hex-encoded email address of person who create this rule -// WHEN: seconds since epoch when this rule was submitted -// COMMENT: hex-encoded string arbitrary comment - - private final static char SEPARATOR = ':'; - private final static String ENCODING = "UTF-8"; - - private final static int TYPE_ROBOTS = 0; - private final static int TYPE_NOROBOTS = 1; - private final static int TYPE_INCLUDE = 2; - private final static int TYPE_EXCLUDE = 3; - - private final static int MOD_ADD = 0; - private final static int MOD_DELETE = 1; - - private String startDateStr; - private String endDateStr; - private int type; - private int mod; - private String who; - private long when; - private String why; - - /** - * @return string "key" including start, end and type information - */ - public String key() { - return startDateStr + SEPARATOR + endDateStr + SEPARATOR + type; - } - - /** - * sets type to Robots - */ - public void setRobots() { - type = TYPE_ROBOTS; - } - /** - * sets type to NoRobots - */ - public void setNoRobots() { - type = TYPE_NOROBOTS; - } - /** - * sets type to Include - */ - public void setInclude() { - type = TYPE_INCLUDE; - } - /** - * sets type to Exclude - */ - public void setExclude() { - type = TYPE_EXCLUDE; - } - /** - * sets mod to ADD - */ - public void setAdd() { - mod = MOD_ADD; - } - /** - * sets mod to DELETE - */ - public void setDelete() { - mod = MOD_DELETE; - } - - /** - * extract values from this object into encoded String representation - * - * @return String representation of values in this object - */ - public String encode() { - StringBuilder encoded = new StringBuilder(100); - try { - encoded.append(startDateStr).append(SEPARATOR); - encoded.append(endDateStr).append(SEPARATOR); - encoded.append(type).append(SEPARATOR); - encoded.append(mod).append(SEPARATOR); - encoded.append(URLEncoder.encode(who,ENCODING)).append(SEPARATOR); - encoded.append(when).append(SEPARATOR); - encoded.append(URLEncoder.encode(why,ENCODING)); - } catch (UnsupportedEncodingException e) { - // this should not happen with a hard-coded UTF-8... - e.printStackTrace(); - } - return encoded.toString(); - } - - /** - * set all values from encoded String version - * - * @param encoded - * @throws ParseException - */ - public void decode(final String encoded) throws ParseException { - String parts[] = encoded.split(String.valueOf(SEPARATOR)); - if(parts.length != 7) { - throw new ParseException("Unable decode (" + encoded + ")",0); - } - startDateStr = Timestamp.padStartDateStr(parts[0]); - endDateStr = Timestamp.padStartDateStr(parts[1]); - type = Integer.valueOf(parts[2]).intValue(); - if(type < TYPE_ROBOTS || type > TYPE_EXCLUDE) { - throw new ParseException("bad type in (" + encoded + ")",3); - } - mod = Integer.valueOf(parts[3]).intValue(); - if(mod < MOD_ADD || mod > MOD_DELETE) { - throw new ParseException("bad mod in (" + encoded + ")",4); - } - try { - who = URLDecoder.decode(parts[4],ENCODING); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - throw new ParseException(e.getMessage(),5); - } - when = Long.valueOf(parts[5]).longValue(); - try { - why = URLDecoder.decode(parts[6],ENCODING); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - throw new ParseException(e.getMessage(),7); - } - } - - /** - * @return Returns the when. - */ - public long getWhen() { - return when; - } - /** - * @param when The when to set. - */ - public void setWhen(long when) { - this.when = when; - } - /** - * @return Returns the who. - */ - public String getWho() { - return who; - } - /** - * @param who The who to set. - */ - public void setWho(String who) { - this.who = who; - } - /** - * @return Returns the why. - */ - public String getWhy() { - return why; - } - /** - * @param why The why to set. - */ - public void setWhy(String why) { - this.why = why; - } - /** - * @return true if this is an ADD rule - */ - public boolean isAdd() { - return mod == MOD_ADD; - } - /** - * @return true if this is a DELETE rule - */ - public boolean isDelete() { - return mod == MOD_DELETE; - } - /** - * @return true if this is a ROBOTS rule - */ - public boolean isRobots() { - return type == TYPE_ROBOTS; - } - /** - * @return true if this is a NOROBOTS rule - */ - public boolean isNoRobots() { - return type == TYPE_NOROBOTS; - } - /** - * @return true if this is an INCLUDE rule - */ - public boolean isInclude() { - return type == TYPE_INCLUDE; - } - /** - * @return true if this is an EXCLUDE rule - */ - public boolean isExclude() { - return type == TYPE_EXCLUDE; - } - - /** - * @return String user friendly version of the mod - */ - public String getPrettyMod() { - if(mod == MOD_ADD) { - return "add"; - } - return "delete"; - } - /** - * @return pretty String representation of the Type - */ - public String getPrettyType() { - String prettyType = null; - switch (type) { - case TYPE_INCLUDE: - prettyType = "Include"; - break; - case TYPE_EXCLUDE: - prettyType = "Exclude"; - break; - case TYPE_ROBOTS: - prettyType = "Use Robots"; - break; - case TYPE_NOROBOTS: - prettyType = "No Robots"; - break; - default: - break; - } - return prettyType; - } - - /** - * @return Returns the endDateStr. - */ - public String getEndDateStr() { - return endDateStr; - } - - /** - * @return Returns pretty version of the endDateStr. - */ - public String getPrettyEndDateStr() { - // TODO: Localization - return Timestamp.parseAfter(endDateStr).prettyDateTime(); - } - /** - * @return Returns pretty version of the startDateStr. - */ - public String getPrettyStartDateStr() { - // TODO: Localization - return Timestamp.parseBefore(startDateStr).prettyDateTime(); - } - - /** - * @param endDateStr The endDateStr to set. - */ - public void setEndDateStr(String endDateStr) { - this.endDateStr = endDateStr; - } - - /** - * @return Returns the startDateStr. - */ - public String getStartDateStr() { - return startDateStr; - } - - /** - * @param startDateStr The startDateStr to set. - */ - public void setStartDateStr(String startDateStr) { - this.startDateStr = startDateStr; - } - - /** - * @param arg a dateString (possibly < 14 digits) - * @return true if arg is in range to which this rule applies - */ - public boolean appliesToDateStr(final String arg) { - return (arg.compareTo(startDateStr.substring(0,arg.length())) >= 0 ) - && (arg.compareTo(endDateStr.substring( 0,arg.length())) <= 0 ); - } - -} Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionRules.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionRules.java 2007-07-26 21:31:53 UTC (rev 1892) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionRules.java 2007-07-26 21:42:35 UTC (rev 1893) @@ -1,231 +0,0 @@ -/* AdministrativeExclusionRules - * - * $Id$ - * - * Created on 12:28:48 PM May 11, 2006. - * - * Copyright (C) 2006 Internet Archive. - * - * This file is part of wayback. - * - * wayback is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * wayback is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with wayback; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ -package org.archive.wayback.accesscontrol; - -import java.text.ParseException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; - -/** - * - * - * @author brad - * @version $Date$, $Revision$ - */ -public class AdministrativeExclusionRules { - - private static String DELIMITER = ","; - private ArrayList<AdministrativeExclusionRule> rules = null; - private String surtPrefix; - /** - * @param surtPrefix - */ - public AdministrativeExclusionRules(String surtPrefix) { - this.surtPrefix = surtPrefix; - } - /** - * prune down rules to only those which apply for a particular timestamp - * first eliminating those outside the timestamp range, and then removing - * ADD which have a (subsequent) DELETE - * - * @param dateStr - * @return ArrayList of applicable and current rules for dateStr - */ - public ArrayList<AdministrativeExclusionRule> filterRules(final String - dateStr) { - - if(rules == null) { - return new ArrayList<AdministrativeExclusionRule>(); - } - - // first separate the rules into ADD and DELETE, only keeping the newest - // for any START-END-TYPE - - HashMap<String, AdministrativeExclusionRule> adds = - new HashMap<String, AdministrativeExclusionRule>(); - - HashMap<String, AdministrativeExclusionRule> deletes = - new HashMap<String, AdministrativeExclusionRule>(); - HashMap<String, AdministrativeExclusionRule> cur = null; - - for(int i = 0; i < rules.size(); i++) { - AdministrativeExclusionRule rule = rules.get(i); - if(!rule.appliesToDateStr(dateStr)) { - continue; - } - String key = rule.key(); - if(rule.isAdd()) { - cur = adds; - } else { - cur = deletes; - } - if(cur.containsKey(key)) { - AdministrativeExclusionRule last = - (AdministrativeExclusionRule) cur.get(key); - if(rule.getWhen() > last.getWhen()) { - cur.put(key,rule); - } - } else { - cur.put(key,rule); - } - } - - // now, remove any ADD for which there is a later DELETE: - - Iterator<AdministrativeExclusionRule> itr = deletes.values().iterator(); - while(itr.hasNext()) { - AdministrativeExclusionRule deleteRule = itr.next(); - String key = deleteRule.key(); - if(adds.containsKey(key)) { - AdministrativeExclusionRule addRule = adds.get(key); - if(deleteRule.getWhen() > addRule.getWhen()) { - adds.remove(key); - } - } - } - - // now the "adds" HashMap contains only rules that apply now, for the - // current time we are interested in. - - return new ArrayList<AdministrativeExclusionRule>(adds.values()); - } - - /** - * finds the most applicable rule for the date in question, and returns it - * - * @param dateStr - * @return most applicable AdministrativeExclusionRule, or null if none - * applied - */ - public AdministrativeExclusionRule getApplicableRule(final String dateStr) { - ArrayList<AdministrativeExclusionRule> applicable = filterRules(dateStr); - // first look for Excludes: - Iterator<AdministrativeExclusionRule> itr = applicable.iterator(); - while(itr.hasNext()) { - AdministrativeExclusionRule rule = itr.next(); - if(rule.isExclude()) { - return rule; - } - } - // then Includes: - itr = applicable.iterator(); - while(itr.hasNext()) { - AdministrativeExclusionRule rule = - (AdministrativeExclusionRule) itr.next(); - if(rule.isInclude()) { - return rule; - } - } - // then NoRobots: - itr = applicable.iterator(); - while(itr.hasNext()) { - AdministrativeExclusionRule rule = - (AdministrativeExclusionRule) itr.next(); - if(rule.isNoRobots()) { - return rule; - } - } - // then Robots: - itr = applicable.iterator(); - while(itr.hasNext()) { - AdministrativeExclusionRule rule = - (AdministrativeExclusionRule) itr.next(); - if(rule.isRobots()) { - return rule; - } - } - // nothing: - return null; - } - - /** - * load rules found in the encoded string argument - * - * @param encodedRules - */ - public void loadRules(final String encodedRules) { - rules = new ArrayList<AdministrativeExclusionRule>(); - String ruleChunks[] = encodedRules.split(DELIMITER); - for(int i = 0; i < ruleChunks.length; i++) { - AdministrativeExclusionRule rule = parseRule(ruleChunks[i]); - if(rule != null) { - rules.add(rule); - } - } - } - - /** - * @param rule - */ - public void addRule(AdministrativeExclusionRule rule) { - if(rules == null) { - rules = new ArrayList<AdministrativeExclusionRule>(); - } - rules.add(rule); - } - - /** - * @return String encoded version of the rules. - */ - public String encodeRules() { - if(rules == null) { - return ""; - } - StringBuilder builder = new StringBuilder(rules.size() * 120); - Iterator<AdministrativeExclusionRule> itr = rules.iterator(); - while(itr.hasNext()) { - AdministrativeExclusionRule rule = itr.next(); - if(builder.length() > 0) { - builder.append(DELIMITER); - } - builder.append(rule.encode()); - } - return builder.toString(); - } - - private AdministrativeExclusionRule parseRule(final String encoded) { - AdministrativeExclusionRule rule = new AdministrativeExclusionRule(); - try { - rule.decode(encoded); - return rule; - } catch (ParseException e) { - e.printStackTrace(); - } - return null; - } - /** - * @return Returns the surtPrefix. - */ - public String getSurtPrefix() { - return surtPrefix; - } - /** - * @return Returns the rules. - */ - public ArrayList<AdministrativeExclusionRule> getRules() { - return rules; - } -} Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionServlet.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionServlet.java 2007-07-26 21:31:53 UTC (rev 1892) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionServlet.java 2007-07-26 21:42:35 UTC (rev 1893) @@ -1,525 +0,0 @@ -/* AdministrativeExclusionServlet - * - * $Id$ - * - * Created on 4:31:48 PM May 12, 2006. - * - * Copyright (C) 2006 Internet Archive. - * - * This file is part of wayback. - * - * wayback is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * wayback is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with wayback; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ -package org.archive.wayback.accesscontrol; - -import java.io.IOException; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.Date; -import java.util.Iterator; -import java.util.Map; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.commons.httpclient.URIException; -import org.archive.wayback.core.Timestamp; -import org.archive.wayback.surt.SURTTokenizer; -import org.archive.wayback.webapp.ServletRequestContext; - -import com.sleepycat.je.DatabaseException; - -/** - * Servlet responsible for UI generation of the Administrative Exclustion - * system. - * - * Primarily this includes: - * 1) generating query form - * 2) displaying current exclusion rules based on queries - * 3) recieving POST requests from clients to add rules - * - * @author brad - * @version $Date$, $Revision$ - */ -public class AdministrativeExclusionServlet extends ServletRequestContext { - - private static final String DEFAULT_USER_AGENT = "ia_archiver"; - private static final String HTML_BR = "<br></br>"; - private static final long serialVersionUID = 1L; - - private static String FORM_OPERATION = "operation"; - - private static String MODIFY_FORM_START = "start"; - private static String MODIFY_FORM_END = "end"; - private static String MODIFY_FORM_URL = "modify_url"; - private static String MODIFY_FORM_SURT = "modify_surt"; - private static String MODIFY_FORM_TYPE = "type"; - private static String MODIFY_FORM_MOD = "mod"; - private static String MODIFY_FORM_WHO = "who"; - private static String MODIFY_FORM_WHY = "why"; - private static String MODIFY_FORM_PREFIX = "prefix"; - - - private static String DELETE_FORM_OPERATION = "delete"; - private static String MODIFY_FORM_OPERATION = "modify"; - - private static String MODIFY_FORM_MOD_ADD_VALUE = "add"; - private static String MODIFY_FORM_MOD_DELETE_VALUE = "delete"; - - private static String MODIFY_FORM_TYPE_EXCLUDE_VALUE = "exclude"; - private static String MODIFY_FORM_TYPE_INCLUDE_VALUE = "include"; - private static String MODIFY_FORM_TYPE_NOROBOTS_VALUE = "norobots"; - private static String MODIFY_FORM_TYPE_ROBOTS_VALUE = "robots"; - - private static String MODIFY_FORM_PREFIX_YES_VALUE = "yes"; - - private static String QUERY_FORM_URL = "query_url"; - private static String QUERY_FORM_ALL = "query_all"; - private static String QUERY_FORM_ALL_VALUE = "yes"; - private static String QUERY_FORM_OPERATION = "query"; - - private static String CHECK_FORM_URL = "check_url"; - private static String CHECK_FORM_TIMESTAMP = "timestamp"; - private static String CHECK_FORM_OPERATION = "check"; - - private static final int RULE_STATUS_HEADER = -1; - private static final int RULE_STATUS_ACTIVE = 0; - private static final int RULE_STATUS_INACTIVE = 1; - private static final int RULE_STATUS_DELETE = 2; - - private AdministrativeExclusionAuthority exclusionAuthority = null; - - private void showPage(HttpServletResponse response, StringBuilder page) - throws IOException { - response.setContentType("text/html"); - response.getOutputStream().print(page.toString()); - } - - private String formatException(Exception e) { - return e.getMessage(); - } - - public boolean handleRequest(HttpServletRequest httpRequest, - HttpServletResponse httpResponse) throws ServletException, - IOException { - AdministrativeExclusionAuthority exclAuth = exclusionAuthority; - - StringBuilder page = new StringBuilder(1024); - page.append("<html><head><title>Wayback: Administrative Exclusions</title></head><body>"); - @SuppressWarnings("unchecked") - Map<String,String[]> queryArgs = httpRequest.getParameterMap(); - String operation = getMapParam(queryArgs,FORM_OPERATION); - if(operation == null) { - page.append(makeCheckForm(queryArgs)); - page.append(makeQueryForm(queryArgs)); - page.append(makeCreateForm(queryArgs)); - } else if(operation.equals(QUERY_FORM_OPERATION)) { - page.append(makeCheckForm(queryArgs)); - page.append(makeQueryForm(queryArgs)); - try { - page.append(handleRuleQuery(exclAuth,queryArgs)); - } catch (URIException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - page.append(formatException(e)); - } catch (ParseException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - page.append(formatException(e)); - } catch (DatabaseException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - page.append(formatException(e)); - } - page.append(makeCreateForm(queryArgs)); - } else if(operation.equals(CHECK_FORM_OPERATION)) { - page.append(makeCheckForm(queryArgs)); - try { - page.append(handleRuleCheck(exclAuth,queryArgs)); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - page.append(formatException(e)); - } - page.append(makeQueryForm(queryArgs)); - page.append(makeCreateForm(queryArgs)); - - } else if((operation.equals(MODIFY_FORM_OPERATION) - || operation.equals(DELETE_FORM_OPERATION))) { - - try { - handleRuleCreate(exclAuth,queryArgs); - page.append("OK - added rule"); - } catch (URIException e) { - e.printStackTrace(); - page.append(formatException(e)); - } catch (ParseException e) { - e.printStackTrace(); - page.append(formatException(e)); - } catch (DatabaseException e) { - e.printStackTrace(); - page.append(formatException(e)); - } - - } else { - page.append("Unknown operation"); - } - page.append("</body></html>"); - showPage(httpResponse,page); - return true; - } - - // HTML GENERATION METHODS: - - private String htmlEncode(String orig) { - return orig; - } - private String propertyHTMLEncode(String orig) { - return orig; - } - - private String makeFormTextInput(String label, String name, - Map<String,String[]> queryArgs, - String suffix) { - String value = ""; - if(queryArgs != null) { - value = propertyHTMLEncode(getMapParamOrEmpty(queryArgs,name)); - } - return label + ":<input type=\"text\" name=\"" + name + "\" VALUE=\"" + - value + "\"></input>" + suffix; - } - private String makeFormTextAreaInput(String label, String name, - Map<String,String[]> queryArgs, String suffix) { - String value = ""; - if(queryArgs != null) { - value = htmlEncode(getMapParamOrEmpty(queryArgs,name)); - } - return label + ":<textarea cols=80 rows=4 name=\"" + name + "\">" + - value + "</textarea>" + suffix; - } - - private String makeFormCheckInput(String label, String name, String value, - Map<String,String[]> queryArgs, String suffix) { - String curValue = getMapParam(queryArgs,name); - String checked = ""; - if(curValue != null && curValue.equals(value)) { - checked = "checked"; - } - return label + ":<input type=\"checkbox\" name=\"" + name + "\" VALUE=\"" + - value + "\" " + checked + "></input>" + suffix; - } - - private String makeHiddenInput(String name, String value) { - return "<input type=\"hidden\" name=\"" + name + "\" value=\"" + - propertyHTMLEncode(value) + "\"></input>"; - } - - - private String makeForm(String method, String submitValue, String content) { - StringBuilder sb = new StringBuilder(1024); - sb.append("<form method=\"").append(method); - sb.append("\" action=\"admin-exclusion\">\n"); - sb.append(content); - sb.append("<input type=\"SUBMIT\" name=\"").append(FORM_OPERATION); - sb.append("\" value=\"").append(submitValue).append("\"></input>\n"); - sb.append("</form>"); - return sb.toString(); - } - private String makeHeaderForm(String title,String method, String submitValue, - String content) { - return "<hr></hr><p><h2>"+title+"</h2>" + makeForm(method,submitValue,content) + "</p>"; - } - - // CHECK FORM AND HANDLING - - private String makeCheckForm(Map<String,String[]> queryArgs) { - String content = makeFormTextInput("Url",CHECK_FORM_URL,queryArgs," ") + - makeFormTextInput("Timestamp",CHECK_FORM_TIMESTAMP,queryArgs," "); - return makeHeaderForm("Test Exclusion","GET",CHECK_FORM_OPERATION,content); - } - - private String handleRuleCheck(AdministrativeExclusionAuthority excl, - Map<String,String[]> queryArgs) throws Exception { - String url = getRequiredMapParam(queryArgs,CHECK_FORM_URL); - String timestamp = getRequiredMapParam(queryArgs,CHECK_FORM_TIMESTAMP); - String userAgent = DEFAULT_USER_AGENT; - - ExclusionResponse response = excl.checkExclusion(userAgent,url,timestamp); - return response.getContentText(); - } - - // RULE QUERY FORM AND HANDLING - - private String makeQueryForm(Map<String,String[]> queryArgs) { - StringBuilder content = new StringBuilder(1024); - content.append(makeFormTextInput("Url",QUERY_FORM_URL,queryArgs,"")); - content.append(makeFormCheckInput("all",QUERY_FORM_ALL,QUERY_FORM_ALL_VALUE,queryArgs,"")); - return makeHeaderForm("Rule Query","GET",QUERY_FORM_OPERATION,content.toString()); - } - - private String handleRuleQuery(AdministrativeExclusionAuthority excl, - Map<String,String[]> queryArgs) throws ParseException, URIException, - DatabaseException { - String url = getRequiredMapParam(queryArgs,QUERY_FORM_URL); - String all = getMapParam(queryArgs,QUERY_FORM_ALL); - boolean showAll = (all != null && all.equals(QUERY_FORM_ALL_VALUE)); - String surt = SURTTokenizer.prefixKey(url); - ArrayList<AdministrativeExclusionRules> matching = excl.matchRules(surt); - StringBuilder matchHTML = new StringBuilder(); - rulesToTable(matchHTML,matching,url,showAll); - return matchHTML.toString(); - } - - private String ruleToType(AdministrativeExclusionRule rule) { - String type = MODIFY_FORM_TYPE_EXCLUDE_VALUE; - if(rule.isExclude()) { - type = MODIFY_FORM_TYPE_EXCLUDE_VALUE; - } else if(rule.isInclude()) { - type = MODIFY_FORM_TYPE_INCLUDE_VALUE; - } else if(rule.isNoRobots()) { - type = MODIFY_FORM_TYPE_NOROBOTS_VALUE; - } else if(rule.isRobots()) { - type = MODIFY_FORM_TYPE_ROBOTS_VALUE; - } - return type; - } - - // RULE DELETION/CREATION FORM AND HANDLING - - private String makeDeleteForm(String surt, AdministrativeExclusionRule rule) { - StringBuilder sb = new StringBuilder(1024); - sb.append(makeHiddenInput(MODIFY_FORM_SURT,surt)); - sb.append(makeHiddenInput(MODIFY_FORM_START,rule.getStartDateStr())); - sb.append(makeHiddenInput(MODIFY_FORM_END,rule.getEndDateStr())); - sb.append(makeHiddenInput(MODIFY_FORM_TYPE,ruleToType(rule))); - sb.append(makeHiddenInput(MODIFY_FORM_MOD,MODIFY_FORM_MOD_DELETE_VALUE)); - sb.append(makeFormTextInput("Who",MODIFY_FORM_WHO,null," ")); - sb.append(makeFormTextInput("Why",MODIFY_FORM_WHY,null," ")); - return makeForm("POST", DELETE_FORM_OPERATION, sb.toString()); - } - - private String makeCreateForm(Map<String,String[]> queryArgs) { - StringBuilder sb = new StringBuilder(1024); - sb.append(makeFormTextInput("Url",MODIFY_FORM_URL,queryArgs,"")); - sb.append(makeFormCheckInput("(prefix)",MODIFY_FORM_PREFIX, - MODIFY_FORM_PREFIX_YES_VALUE,queryArgs,HTML_BR)); - - sb.append(makeFormTextInput("Start",MODIFY_FORM_START,queryArgs,"")); - sb.append(makeFormTextInput("End",MODIFY_FORM_END,queryArgs,HTML_BR)); - - sb.append("Type: <select name=\""+MODIFY_FORM_TYPE+"\">" + - "<option>"+MODIFY_FORM_TYPE_EXCLUDE_VALUE+"</option>" + - "<option>"+MODIFY_FORM_TYPE_INCLUDE_VALUE+"</option>" + - "<option>"+MODIFY_FORM_TYPE_ROBOTS_VALUE+"</option>" + -// "<option>"+MODIFY_FORM_TYPE_NOROBOTS_VALUE+"</option>" + - "</select>").append(HTML_BR).append("\n"); - -// sb.append("Add/Remove: <select NAME=\""+MODIFY_FORM_MOD+"\">" + -// "<option>"+MODIFY_FORM_MOD_ADD_VALUE+"</option>" + -// "<option>"+MODIFY_FORM_MOD_DELETE_VALUE+"</option>" + -// "</select><br></br>\n"); - sb.append(makeHiddenInput(MODIFY_FORM_MOD,MODIFY_FORM_MOD_ADD_VALUE)); - - sb.append(makeFormTextInput("Who",MODIFY_FORM_WHO,queryArgs,HTML_BR)); - sb.append(makeFormTextAreaInput("Why",MODIFY_FORM_WHY,queryArgs,HTML_BR)); - - return makeHeaderForm("Compose Rule","POST",MODIFY_FORM_OPERATION, - sb.toString()); - } - - private void handleRuleCreate(AdministrativeExclusionAuthority auth, - Map<String,String[]> queryMap) throws ParseException, URIException, DatabaseException { - - AdministrativeExclusionRule rule = new AdministrativeExclusionRule(); - - String start = getRequiredMapParam(queryMap,MODIFY_FORM_START); - String startDateStr = Timestamp.padStartDateStr(start); - if(!startDateStr.equals(start)) { - throw new ParseException("invalid value: " + MODIFY_FORM_START,0); - } - - String end = getRequiredMapParam(queryMap,MODIFY_FORM_END); - String endDateStr = Timestamp.padEndDateStr(end); - if(!endDateStr.equals(end)) { - throw new ParseException("invalid value: " + MODIFY_FORM_END,0); - } - - String type = getRequiredMapParam(queryMap,MODIFY_FORM_TYPE); - if(type.equals(MODIFY_FORM_TYPE_EXCLUDE_VALUE)) { - rule.setExclude(); - } else if(type.equals(MODIFY_FORM_TYPE_INCLUDE_VALUE)) { - rule.setInclude(); - } else if(type.equals(MODIFY_FORM_TYPE_ROBOTS_VALUE)) { - rule.setRobots(); - } else if(type.equals(MODIFY_FORM_TYPE_NOROBOTS_VALUE)) { - rule.setNoRobots(); - } else { - throw new ParseException("invalid value: " + MODIFY_FORM_TYPE,0); - } - - String mod = getRequiredMapParam(queryMap,MODIFY_FORM_MOD); - if(mod.equals(MODIFY_FORM_MOD_ADD_VALUE)) { - rule.setAdd(); - } else if(mod.equals(MODIFY_FORM_MOD_DELETE_VALUE)) { - rule.setDelete(); - } else { - throw new ParseException("invalid value: " + MODIFY_FORM_MOD,0); - } - - String who = getRequiredMapParam(queryMap,MODIFY_FORM_WHO); - String why = getRequiredMapParam(queryMap,MODIFY_FORM_WHY); - - String url = getMapParam(queryMap,MODIFY_FORM_URL); - String surt = getMapParam(queryMap,MODIFY_FORM_SURT); - if(surt == null) { - if(url == null) { - throw new ParseException("Missing argument " + MODIFY_FORM_URL,0); - } - String prefix = getMapParam(queryMap,MODIFY_FORM_PREFIX); - if(prefix == null || !prefix.equals(MODIFY_FORM_PREFIX_YES_VALUE)) { - surt = SURTTokenizer.exactKey(url); - } else { - surt = SURTTokenizer.prefixKey(url); - } - } - rule.setWho(who); - rule.setWhy(why); - rule.setWhen(new Date().getTime()); - rule.setStartDateStr(startDateStr); - rule.setEndDateStr(endDateStr); - - auth.addRuleFor(surt,rule); - } - - - private int statusForRule(AdministrativeExclusionRule rule, - AdministrativeExclusionRules rules) { - int status = RULE_STATUS_ACTIVE; - if(rule.isDelete()) { - status = RULE_STATUS_DELETE; - } else { - // has rule been deleted? - String key = rule.key(); - boolean deleted = false; - - Iterator<AdministrativeExclusionRule> itr = - rules.getRules().iterator(); - while(itr.hasNext()) { - AdministrativeExclusionRule daRule = - (AdministrativeExclusionRule) itr.next(); - if(daRule.isDelete() && key.equals(daRule.key()) && - daRule.getWhen() > rule.getWhen()) { - deleted = true; - break; - } - } - - if(deleted) { - status = RULE_STATUS_INACTIVE; - } - } - return status; - } - - private void tableCell(StringBuilder page,String content, int status) { - content = htmlEncode(content); - String style = ""; - if(status == RULE_STATUS_DELETE) { - style = "style=\"color:red;\""; - } else if (status == RULE_STATUS_INACTIVE) { - style = "style=\"text-decoration:line-through;\""; - } else if (status == RULE_STATUS_HEADER) { - style = "style=\"bold:yes;\""; - } - page.append("<td ").append(style).append(">"); - page.append(htmlEncode(content)); - page.append("</td>"); - } - - private void ruleHeaderRow(StringBuilder page) { - int status = RULE_STATUS_HEADER; - page.append("<tr>"); - tableCell(page,"Url Prefix",status); - tableCell(page,"Starts",status); - tableCell(page,"Ends",status); - tableCell(page,"Rule Type",status); - tableCell(page,"Add/Delete",status); - tableCell(page,"When",status); - tableCell(page,"Admin",status); - tableCell(page,"Comment",status); - page.append("</tr>\n"); - } - - private void ruleToRow(StringBuilder page, String surt, - AdministrativeExclusionRule rule,int status) { - String surtPrefix = surt; - if(surtPrefix.endsWith("\t")) { - surtPrefix = surtPrefix.substring(0,surtPrefix.length()-1); - } else { - surtPrefix = surtPrefix + "*"; - } - - page.append("<tr>"); - tableCell(page,surtPrefix,status); - tableCell(page,rule.getPrettyStartDateStr(),status); - tableCell(page,rule.getPrettyEndDateStr(),status); - tableCell(page,rule.getPrettyType(),status); - tableCell(page,rule.getPrettyMod(),status); - int sse = (int) (rule.getWhen()/1000); - // TODO: Localization - tableCell(page, Timestamp.fromSse(sse).prettyDateTime(), status); - tableCell(page,rule.getWho(),status); - // TODO: shrink - if(status == RULE_STATUS_ACTIVE) { - page.append("<td>").append(rule.getWhy()).append(" "). - append(makeDeleteForm(surt,rule)).append("</td>"); - } else { - tableCell(page,rule.getWhy(),status); - } - page.append("</tr>\n"); - } - - private void rulesToTable(StringBuilder page, - ArrayList<AdministrativeExclusionRules> matching, String url, - boolean showAll) { - - if(matching.size() > 0) { - page.append("<table border=1>"); - ruleHeaderRow(page); - Iterator<AdministrativeExclusionRules> itr = matching.iterator(); - while(itr.hasNext()) { - AdministrativeExclusionRules rules = itr.next(); - - String surtPrefix = rules.getSurtPrefix(); - Iterator<AdministrativeExclusionRule> ruleItr = null; - if(showAll) { - ruleItr = rules.getRules().iterator(); - } else { - ruleItr = rules.filterRules("").iterator(); - } - while(ruleItr.hasNext()) { - AdministrativeExclusionRule rule = - (AdministrativeExclusionRule) ruleItr.next(); - int status = statusForRule(rule,rules); - ruleToRow(page,surtPrefix,rule, status); - } - } - page.append("</table>\n"); - } else { - page.append("<p>No records match("+url+")</p>\n"); - } - } -} Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionAuthority.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionAuthority.java 2007-07-26 21:31:53 UTC (rev 1892) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionAuthority.java 2007-07-26 21:42:35 UTC (rev 1893) @@ -1,46 +0,0 @@ -/* ExclusionAuthority - * - * $Id$ - * - * Created on 3:34:19 PM May 9, 2006. - * - * Copyright (C) 2006 Internet Archive. - * - * This file is part of wayback. - * - * wayback is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * wayback is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with wayback; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ -package org.archive.wayback.accesscontrol; - -/** - * - * - * @author brad - * @version $Date$, $Revision$ - */ -public interface ExclusionAuthority { - /** - * determines whether userAgent can view urlString for captureDate, - * encapsulating the response in a returned ExclusionResponse object - * - * @param userAgent - * @param urlString - * @param captureDate - * @return ExclusionResponse with answer to the query - * @throws Exception - */ - public ExclusionResponse checkExclusion(String userAgent, String urlString, - String captureDate) throws Exception; -} Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionResponse.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionResponse.java 2007-07-26 21:31:53 UTC (rev 1892) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionResponse.java 2007-07-26 21:42:35 UTC (rev 1893) @@ -1,281 +0,0 @@ -/* ExclusionResponse - * - * $Id$ - * - * Created on 4:30:17 PM Feb 13, 2006. - * - * Copyright (C) 2006 Internet Archive. - * - * This file is part of wayback. - * - * wayback is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * wayback is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with wayback; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ -package org.archive.wayback.accesscontrol; - - -import java.io.IOException; -import java.io.OutputStream; - -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerConfigurationException; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; - -import org.w3c.dom.Attr; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; - -/** - * Abstraction for the result of an exclusion query, and generating the - * differentiating parts of an HTTP response. The original implementation - * returned XML. Without the XML, this class is really not needed, but as - * I suspect we'll return to an XML HTTML response in the future, including - * this for now. - * - * - * @author brad - * @version $Date$, $Revision$ - */ -public class ExclusionResponse { - - /** - * Reponse is considered "authoritative" - */ - public final static String EXLCUSION_AUTHORITATIVE = "Authoritative"; - /** - * Response is not "authoritative" -- some assumptions may have been made - */ - public final static String EXLCUSION_NON_AUTHORITATIVE = "Non-Authoritative"; - /** - * Response indicates that access is granted - */ - public final static boolean EXLCUSION_AUTHORIZED = true; - /** - * Response indicates that access is not granted - */ - public final static boolean EXLCUSION_NOT_AUTHORIZED = false; - - private final static boolean USE_XML = false; - - private String responseType; - - private String hostname; - - private boolean authorized; - - private String message; - - /** - * Constructor - * - * @param hostname - * @param responseType - * @param authorized - * @param message - */ - public ExclusionResponse(final String hostname, final String responseType, - final boolean authorized,final String message) { - this.hostname = hostname; - this.responseType = responseType; - this.authorized = authorized; - this.message = message; - } - - /** - * Constuctor - * - * @param hostname - * @param responseType - * @param authorized - */ - public ExclusionResponse(final String hostname, final String responseType, - final boolean authorized) { - - this.hostname = hostname; - this.responseType = responseType; - this.authorized = authorized; - this.message = ""; - } - - /** - * Send the HTTP message body to requesting client, via the OutputStream - * - * @param os - */ - public void writeResponse(OutputStream os) { - if(USE_XML) { - writeResponseXML(os); - } else { - writeResponseText(os); - } - } - - /** - * @return String HTTP Content-Type field: "text/xml" or "text/plain" - */ - public String getContentType() { - if(USE_XML) { - return getContentTypeXML(); - } else { - return getContentTypeText(); - } - } - - private String getContentTypeText() { - return "text/plain"; - } - - /** - * @return String version of response - */ - public String getContentText() { - return authorized ? "OK" : "BLOCKED - " + message; - } - - private void writeResponseText(OutputStream os) { - - String content = getContentText(); - try { - os.write(content.getBytes()); - } catch (IOException e) { - // TODO what now? - e.printStackTrace(); - } - } - - private String getContentTypeXML() { - return "text/xml"; - } - - private void writeResponseXML(OutputStream os) { - - Document doc; - try { - doc = getResponseXMLDocument(); - DOMSource source = new DOMSource(doc); - TransformerFactory transFactory = TransformerFactory.newInstance(); - Transformer transformer = transFactory.newTransformer(); - transformer.setOutputProperty("indent", "yes"); - StreamResult result = new StreamResult(os); - transformer.transform(source, result); - } catch (ParserConfigurationException e) { - // TODO is anything output here? - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (TransformerConfigurationException e) { - // TODO is anything output here? - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (TransformerException e) { - // TODO is anything output here? - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - private Document getResponseXMLDocument() throws ParserConfigurationException { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - Document doc = factory.newDocumentBuilder().newDocument(); - Element response = addNode(doc, doc, "response"); - addAttribute(doc,response,"type",responseType); - addAttribute(doc,response,"hostname",hostname); - addAttribute(doc,response,"authorized",authorized?"true":"false"); - addAttribute(doc,response,"message",message); - return doc; - } - - private static Element addNode(Document doc, Node parent, String name) { - Element child = doc.createElement(name); - parent.appendChild(child); - return child; - } - - // All below here stolen from NutchwaxOpenSearchServlet... - -// private static void addNode(Document doc, Node parent, String name, -// String text) { -// Element child = doc.createElement(name); -// child.appendChild(doc.createTextNode(getLegalXml(text))); -// parent.appendChild(child); -// } - - private static void addAttribute(Document doc, Element node, String name, - String value) { - Attr attribute = doc.createAttribute(name); - attribute.setValue(getLegalXml(value)); - node.getAttributes().setNamedItem(attribute); - } - - /* - * Ensure string is legal xml. - * First look to see if string has illegal characters. If it doesn't, - * just return it. Otherwise, create new string with illegal characters - * @param text String to verify. - * @return Passed <code>text</code> or a new string with illegal - * characters removed if any found in <code>text</code>. - * @see http://www.w3.org/TR/2000/REC-xml-20001006#NT-Char - */ - private static String getLegalXml(final String text) { - if (text == null) { - return null; - } - boolean allLegal = true; - for (int i = 0; i < text.length(); i++) { - if (!isLegalXml(text.charAt(i))) { - allLegal = false; - break; - } - } - return allLegal ? text : createLegalXml(text); - } - - private static String createLegalXml(final String text) { - if (text == null) { - return null; - } - StringBuffer buffer = new StringBuffer(text.length()); - for (int i = 0; i < text.length(); i++) { - char c = text.charAt(i); - if (isLegalXml(c)) { - buffer.append(c); - } - } - return buffer.toString(); - } - - private static boolean isLegalXml(final char c) { - return c == 0x9 || c == 0xa || c == 0xd || (c >= 0x20 && c <= 0xd7ff) - || (c >= 0xe000 && c <= 0xfffd) - || (c >= 0x10000 && c <= 0x10ffff); - } - /** - * @return Returns the authorized. - */ - public boolean isAuthorized() { - return authorized; - } - /** - * @return Returns the message. - */ - public String getMessage() { - return message; - } -} Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionServlet.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionServlet.java 2007-07-26 21:31:53 UTC (rev 1892) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionServlet.java 2007-07-26 21:42:35 UTC (rev 1893) @@ -1,137 +0,0 @@ -/* ExclusionServlet - * - * $Id$ - * - * Created on 5:28:10 PM Feb 13, 2006. - * - * Copyright (C) 2006 Internet Archive. - * - * This file is part of wayback. - * - * wayback is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * wayback is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with wayback; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ -package org.archive.wayback.accesscontrol; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.Map; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.archive.wayback.webapp.ServletRequestContext; - -/** - * - * - * @author brad - * @version $Date$, $Revision$ - */ -public class ExclusionServlet extends ServletRequestContext { - private static final long serialVersionUID = 1L; - - private static final String OPERATION_ARGUMENT = "operation"; - - private static final String URL_ARGUMENT = "url"; - - private static final String USER_AGENT_ARGUMENT = "useragent"; - - private static final String TIMESTAMP_ARGUMENT = "timestamp"; - - private static final String OPERATION_CHECK = "check"; - - private ExclusionAuthority exclusionAuthority = null; -// private static final String OPERATION_PURGE = "purge"; - - - public boolean handleRequest(HttpServletRequest httpRequest, - HttpServletResponse httpResponse) throws IOException, - ServletException { - ExclusionAuthority exclAuth = exclusionAuthority; - - @SuppressWarnings("unchecked") - Map<String,String[]> queryArgs = httpRequest.getParameterMap(); - String url = getMapParam(queryArgs, URL_ARGUMENT); - String operation = getMapParam(queryArgs, OPERATION_ARGUMENT); - String userAgent = getMapParam(queryArgs, USER_AGENT_ARGUMENT); - String timestamp = getMapParam(queryArgs, TIMESTAMP_ARGUMENT); - ExclusionResponse eclResponse = null; - if (operation == null) { - eclResponse = new ExclusionResponse("-", "AccessError", false, - "No " + OPERATION_ARGUMENT + " argument supplied"); - } else if (url == null) { - eclResponse = new ExclusionResponse("-", "AccessError", false, - "No " + URL_ARGUMENT + " argument supplied"); - } else { - if (operation.equals(OPERATION_CHECK)) { - if (userAgent == null) { - eclResponse = new ExclusionResponse("-", "AccessError", - false, "No " + USER_AGENT_ARGUMENT - + " argument supplied"); - } else if (timestamp == null) { - eclResponse = new ExclusionResponse("-", "AccessError", - false, "No " + TIMESTAMP_ARGUMENT - + " argument supplied"); - } else { - try { - - eclResponse = exclAuth.checkExclusion(userAgent, url, - timestamp); - - } catch (Exception e) { - e.printStackTrace(); - eclResponse = new ExclusionResponse("-", "ServerError", - false, e.getMessage()); - } - } - // } else if(operation.equals(OPERATION_PURGE)) { - // try { - // eclResponse = roboCache.purgeUrl(url); - // } catch (MalformedURLException e) { - // // TODO Auto-generated catch block - // e.printStackTrace(); - // eclResponse = new ExclusionResponse("-","ServerError", - // false,e.getMessage()); - // } catch (DatabaseException e) { - // // TODO Auto-generated catch block - // e.printStackTrace(); - // eclResponse = new ExclusionResponse("-","ServerError", - // false,e.getMessage()); - // } - } else { - eclResponse = new ExclusionResponse("-", "AccessError", false, - "Unknown " + OPERATION_ARGUMENT); - } - } - httpResponse.setContentType(eclResponse.getContentType()); - OutputStream os = httpResponse.getOutputStream(); - eclResponse.writeResponse(os); - return true; - } - - /** - * @param exclusionAuthority the exclusionAuthority to set - */ - public void setExclusionAuthority(ExclusionAuthority exclusionAuthority) { - this.exclusionAuthority = exclusionAuthority; - } - - /** - * @return the exclusionAuthority - */ - public ExclusionAuthority getExclusionAuthority() { - return exclusionAuthority; - } -} Copied: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/AdministrativeExclusionAuthority.java (from rev 1880, trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/AdministrativeExclusionAuthority.java) =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/AdministrativeExclusionAuthority.java (rev 0) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/AdministrativeExclusionAuthority.java 2007-07-26 21:42:35 UTC (rev 1893) @@ -0,0 +1,169 @@ +/* AdministrativeExclusionAuthority + * + * $Id$ + * + * Created on 2:47:39 PM May 10, 2006. + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of wayback. + * + * wayback is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * wayback is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with wayback; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Su... [truncated message content] |
From: <bra...@us...> - 2008-03-01 02:05:24
|
Revision: 2211 http://archive-access.svn.sourceforge.net/archive-access/?rev=2211&view=rev Author: bradtofel Date: 2008-02-29 18:05:29 -0800 (Fri, 29 Feb 2008) Log Message: ----------- INITIAL REV: Wayback Exclusion interface to use the access-control "Oracle" Added Paths: ----------- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/oracleclient/ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/oracleclient/OracleExclusionFilter.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/oracleclient/OracleExclusionFilterFactory.java Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/oracleclient/OracleExclusionFilter.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/oracleclient/OracleExclusionFilter.java (rev 0) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/oracleclient/OracleExclusionFilter.java 2008-03-01 02:05:29 UTC (rev 2211) @@ -0,0 +1,71 @@ +package org.archive.wayback.accesscontrol.oracleclient; + +import java.util.Date; + +import org.archive.accesscontrol.AccessControlClient; +import org.archive.accesscontrol.RobotsUnavailableException; +import org.archive.accesscontrol.RuleOracleUnavailableException; +import org.archive.util.ArchiveUtils; +import org.archive.wayback.WaybackConstants; +import org.archive.wayback.core.SearchResult; +import org.archive.wayback.core.Timestamp; +import org.archive.wayback.util.ObjectFilter; + +public class OracleExclusionFilter implements ObjectFilter<SearchResult> { + ObjectFilter<SearchResult> robotFilter = null; + AccessControlClient client = null; + private String accessGroup = null; + + private final static String POLICY_ALLOW = "allow"; + private final static String POLICY_BLOCK = "block"; + private final static String POLICY_ROBOT = "robots"; + + + public OracleExclusionFilter(String oracleUrl, String accessGroup) { + client = new AccessControlClient(oracleUrl); + this.accessGroup = accessGroup; + } + + + public int filterObject(SearchResult o) { + String url = o.get(WaybackConstants.RESULT_URL); + Date captureDate = Timestamp.parseBefore( + o.get(WaybackConstants.RESULT_CAPTURE_DATE)).getDate(); + Date retrievalDate = new Date(); + + String policy; + try { + policy = client.getPolicy(ArchiveUtils.addImpliedHttpIfNecessary(url), captureDate, retrievalDate, + accessGroup); + if(policy != null) { + if(policy.equals(POLICY_ALLOW)) { + return FILTER_INCLUDE; + } else if(policy.equals(POLICY_BLOCK)) { + return FILTER_EXCLUDE; + } else if(policy.equals(POLICY_ROBOT)) { + return FILTER_INCLUDE; +// if(robotFilter != null) { +// return robotFilter.filterObject(o); +// } else { +// return FILTER_EXCLUDE; +// } + } + } + } catch (RobotsUnavailableException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (RuleOracleUnavailableException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return FILTER_EXCLUDE; + } + + public ObjectFilter<SearchResult> getRobotFilter() { + return robotFilter; + } + + public void setRobotFilter(ObjectFilter<SearchResult> robotFilter) { + this.robotFilter = robotFilter; + } +} Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/oracleclient/OracleExclusionFilterFactory.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/oracleclient/OracleExclusionFilterFactory.java (rev 0) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/oracleclient/OracleExclusionFilterFactory.java 2008-03-01 02:05:29 UTC (rev 2211) @@ -0,0 +1,51 @@ +package org.archive.wayback.accesscontrol.oracleclient; + +import org.archive.wayback.accesscontrol.ExclusionFilterFactory; +import org.archive.wayback.accesscontrol.robotstxt.RobotExclusionFilterFactory; +import org.archive.wayback.core.SearchResult; +import org.archive.wayback.util.ObjectFilter; + +public class OracleExclusionFilterFactory implements ExclusionFilterFactory { + + private RobotExclusionFilterFactory robotFactory = null; + private String oracleUrl = null; + private String accessGroup = null; + + public ObjectFilter<SearchResult> get() { + OracleExclusionFilter filter = new OracleExclusionFilter(oracleUrl, + accessGroup); + if(robotFactory != null) { + filter.setRobotFilter(robotFactory.get()); + } + return filter; + } + + public void shutdown() { + // no-op... yet.. + } + + public RobotExclusionFilterFactory getRobotFactory() { + return robotFactory; + } + + public void setRobotFactory(RobotExclusionFilterFactory robotFactory) { + this.robotFactory = robotFactory; + } + + public String getOracleUrl() { + return oracleUrl; + } + + public void setOracleUrl(String oracleUrl) { + this.oracleUrl = oracleUrl; + } + + public String getAccessGroup() { + return accessGroup; + } + + public void setAccessGroup(String accessGroup) { + this.accessGroup = accessGroup; + } + +} This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <bra...@us...> - 2008-07-01 23:23:18
|
Revision: 2354 http://archive-access.svn.sourceforge.net/archive-access/?rev=2354&view=rev Author: bradtofel Date: 2008-07-01 16:23:28 -0700 (Tue, 01 Jul 2008) Log Message: ----------- REFACTOR: SearchResult => (Url|Capture)SearchResult Modified Paths: -------------- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/CompositeExclusionFilterFactory.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionFilterFactory.java trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExternalExcluder.java Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/CompositeExclusionFilterFactory.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/CompositeExclusionFilterFactory.java 2008-07-01 23:21:32 UTC (rev 2353) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/CompositeExclusionFilterFactory.java 2008-07-01 23:23:28 UTC (rev 2354) @@ -27,7 +27,7 @@ import java.util.ArrayList; import java.util.Iterator; -import org.archive.wayback.core.SearchResult; +import org.archive.wayback.core.CaptureSearchResult; import org.archive.wayback.resourceindex.filters.CompositeExclusionFilter; import org.archive.wayback.util.ObjectFilter; @@ -54,7 +54,7 @@ /* (non-Javadoc) * @see org.archive.wayback.resourceindex.ExclusionFilterFactory#get() */ - public ObjectFilter<SearchResult> get() { + public ObjectFilter<CaptureSearchResult> get() { Iterator<ExclusionFilterFactory> itr = factories.iterator(); CompositeExclusionFilter filter = new CompositeExclusionFilter(); while(itr.hasNext()) { Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionFilterFactory.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionFilterFactory.java 2008-07-01 23:21:32 UTC (rev 2353) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionFilterFactory.java 2008-07-01 23:23:28 UTC (rev 2354) @@ -24,7 +24,7 @@ */ package org.archive.wayback.accesscontrol; -import org.archive.wayback.core.SearchResult; +import org.archive.wayback.core.CaptureSearchResult; import org.archive.wayback.util.ObjectFilter; /** * @@ -37,7 +37,7 @@ * @return an ObjectFilter object that filters records based on * some set of exclusion rules */ - public ObjectFilter<SearchResult> get(); + public ObjectFilter<CaptureSearchResult> get(); /** * close any resources used by this ExclusionFilter system. */ Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExternalExcluder.java =================================================================== --- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExternalExcluder.java 2008-07-01 23:21:32 UTC (rev 2353) +++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExternalExcluder.java 2008-07-01 23:23:28 UTC (rev 2354) @@ -24,10 +24,8 @@ */ package org.archive.wayback.accesscontrol; -import org.apache.commons.httpclient.URIException; -import org.archive.net.LaxURI; -import org.archive.wayback.WaybackConstants; -import org.archive.wayback.core.SearchResult; +import org.archive.wayback.core.CaptureSearchResult; +import org.archive.wayback.core.Timestamp; import org.archive.wayback.util.ObjectFilter; import org.springframework.beans.factory.xml.XmlBeanFactory; import org.springframework.core.io.FileSystemResource; @@ -44,12 +42,12 @@ */ public class ExternalExcluder { private static ExclusionFilterFactory factory = null; - private ObjectFilter<SearchResult> filter = null; + private ObjectFilter<CaptureSearchResult> filter = null; private final static String CONFIG_ID = "excluder-factory"; /** * @param filter */ - public ExternalExcluder(ObjectFilter<SearchResult> filter) { + public ExternalExcluder(ObjectFilter<CaptureSearchResult> filter) { this.filter = filter; } /** @@ -58,20 +56,10 @@ * @return true if the url-timestamp should not be shown to end users */ public boolean isExcluded(String urlString, String timestamp) { - SearchResult sr = new SearchResult(); + CaptureSearchResult sr = new CaptureSearchResult(); - LaxURI url = null; - String host = null; - try { - url = new LaxURI(urlString,true); - host = url.getHost(); - } catch (URIException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - return true; - } - sr.put(WaybackConstants.RESULT_ORIG_HOST, host); - sr.put(WaybackConstants.RESULT_URL, urlString); + sr.setOriginalUrl(urlString); + sr.setCaptureTimestamp(Timestamp.parseBefore(timestamp).getDateStr()); int ruling = filter.filterObject(sr); return (ruling != ObjectFilter.FILTER_INCLUDE); This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |