archive-access-cvs Mailing List for Web Archive Access Utilities (Page 11)

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 422-6466

Wayback-1 - Build # 41 - Still Failing:

Check console output at https://builds.archive.org:1443/job/Wayback-1/41/ to view the results.



Revision: 3518
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3518&view=rev
Author:   bradtofel
Date:     2011-09-06 03:52:41 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
LOGGING

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/LocationDBResourceStore.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/LocationDBResourceStore.java
===================================================================

--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/LocationDBResourceStore.java	2011-09-06 03:49:59 UTC (rev 3517)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/LocationDBResourceStore.java	2011-09-06 03:52:41 UTC (rev 3518)
@@ -61,6 +61,7 @@
 			throw new ResourceNotAvailableException(e1.getLocalizedMessage());
 		}
 		if(urls == null || urls.length == 0) {
+			LOGGER.warning("Unable to locate(" + fileName + ")");
 			throw new ResourceNotAvailableException("Unable to locate(" +
 					fileName + ")");
 		}

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Wayback-1 - Build # 40 - Still Failing:

Check console output at https://builds.archive.org:1443/job/Wayback-1/40/ to view the results.



Revision: 3517
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3517&view=rev
Author:   bradtofel
Date:     2011-09-06 03:49:59 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
LOGGING

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FlatFileResourceFileLocationDB.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FlatFileResourceFileLocationDB.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FlatFileResourceFileLocationDB.java	2011-09-06 03:49:28 UTC (rev 3516)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FlatFileResourceFileLocationDB.java	2011-09-06 03:49:59 UTC (rev 3517)
@@ -22,11 +22,14 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.logging.Logger;
 
 import org.archive.wayback.util.CloseableIterator;
 import org.archive.wayback.util.flatfile.FlatFile;
 
 public class FlatFileResourceFileLocationDB implements ResourceFileLocationDB  {
+	private final static Logger LOGGER =
+		Logger.getLogger(FlatFileResourceFileLocationDB.class.getName());
 	private String path = null;
 	private FlatFile flatFile = null;
 	private String delimiter = "\t";
@@ -62,6 +65,9 @@
 			CloseableIterator<String> citr = (CloseableIterator<String>) itr;
 			citr.close();
 		}
+		if(urls.size() == 0) {
+			LOGGER.warning("No locations for " + name + " in " + path);
+		}
 		String[] a = new String[urls.size()];
 		for(int i=0; i < urls.size(); i++) {
 			a[i] = urls.get(i);

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3516
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3516&view=rev
Author:   bradtofel
Date:     2011-09-06 03:49:28 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
LOGGING

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/ResourceFactory.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/ResourceFactory.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/ResourceFactory.java	2011-09-06 03:48:39 UTC (rev 3515)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/ResourceFactory.java	2011-09-06 03:49:28 UTC (rev 3516)
@@ -25,6 +25,7 @@
 import java.io.InputStream;
 import java.io.RandomAccessFile;
 import java.net.URL;
+import java.util.logging.Logger;
 
 import org.archive.io.ArchiveReader;
 import org.archive.io.ArchiveRecord;
@@ -46,14 +47,25 @@
  * @version $Date$, $Revision$
  */
 public class ResourceFactory {
+	private static final Logger LOGGER = 
+		Logger.getLogger(ResourceFactory.class.getName());
 
 	public static Resource getResource(String urlOrPath, long offset)
 	throws IOException, ResourceNotAvailableException {
-		if(urlOrPath.startsWith("http://")) {
-			return getResource(new URL(urlOrPath), offset);
-		} else {
-			// assume local path:
-			return getResource(new File(urlOrPath), offset);
+		LOGGER.info("Fetching: " + urlOrPath + " : " + offset);
+		try {
+			if(urlOrPath.startsWith("http://")) {
+				return getResource(new URL(urlOrPath), offset);
+			} else {
+				// assume local path:
+				return getResource(new File(urlOrPath), offset);
+			}
+		} catch(ResourceNotAvailableException e) {
+			LOGGER.warning("ResourceNotAvailable for " + urlOrPath + " " + e.getMessage());
+			throw e;
+		} catch(IOException e) {
+			LOGGER.warning("ResourceNotAvailable for " + urlOrPath + " " + e.getMessage());
+			throw e;
 		}
 	}
 

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3515
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3515&view=rev
Author:   bradtofel
Date:     2011-09-06 03:48:39 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
BUGFIX: relaxed internal Iterator constraint - no longer must be a CloseableIterator

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/ObjectFilterIterator.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/ObjectFilterIterator.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/ObjectFilterIterator.java	2011-09-06 03:47:31 UTC (rev 3514)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/ObjectFilterIterator.java	2011-09-06 03:48:39 UTC (rev 3515)
@@ -35,7 +35,7 @@
 	Iterator<T> itr = null;
 	boolean aborted = false;
 	T cachedNext = null;
-	public ObjectFilterIterator(CloseableIterator<T> itr, 
+	public ObjectFilterIterator(Iterator<T> itr, 
 			ObjectFilter<T> filter) {
 		this.itr = itr;
 		this.filter = filter;

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3514
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3514&view=rev
Author:   bradtofel
Date:     2011-09-06 03:47:31 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
FEATURE: added simple String.join functionality

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/StringFormatter.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/StringFormatter.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/StringFormatter.java	2011-09-06 03:46:44 UTC (rev 3513)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/StringFormatter.java	2011-09-06 03:47:31 UTC (rev 3514)
@@ -22,6 +22,7 @@
 import java.text.DateFormat;
 import java.text.Format;
 import java.text.MessageFormat;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -224,4 +225,21 @@
 	public String spaceToNBSP(String input) {
 		return input.replaceAll(" ", "&nbsp;");
 	}
+	
+	public static String join(String delim, String...stuff) {
+		int max = stuff.length - 1;
+		int len = delim.length() * max;
+		for(String s : stuff) {
+			len += s.length();
+		}
+		StringBuilder sb = new StringBuilder(len);
+		for(int i = 0; i < stuff.length; i++) {
+			sb.append(stuff[i]);
+			if(i < max) {
+				sb.append(delim);
+			}
+		}
+		return sb.toString();
+	}
+
 }

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3513
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3513&view=rev
Author:   bradtofel
Date:     2011-09-06 03:46:44 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
FEATURE:
   - added configuration to force clean queries, rather than forcing it to be active
   - added configuration of additional AccessPoint specific filter factories for search results
   - implemented AccessPointAware configuration of collaborating components

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/AccessPoint.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/AccessPoint.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/AccessPoint.java	2011-09-06 03:43:17 UTC (rev 3512)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/AccessPoint.java	2011-09-06 03:46:44 UTC (rev 3513)
@@ -102,6 +102,7 @@
 	private boolean serveStatic = true;
 	private boolean bounceToReplayPrefix = false;
 	private boolean bounceToQueryPrefix = false;
+	private boolean forceCleanQueries = false;
 
 	private String liveWebPrefix = null;
 	private String staticPrefix = null;
@@ -130,7 +131,13 @@
 	private ExclusionFilterFactory exclusionFactory = null;
 	private BooleanOperator<WaybackRequest> authentication = null;
 	private long embargoMS = 0;
+	private CustomResultFilterFactory filterFactory = null;
 
+	public void init() {
+		checkAccessPointAware(collection,exception,query,parser,replay,
+				uriConverter,exclusionFactory, authentication, filterFactory);
+	}
+	
 	protected boolean dispatchLocal(HttpServletRequest httpRequest,
 			HttpServletResponse httpResponse) 
 	throws ServletException, IOException {
@@ -284,6 +291,17 @@
 		}
 	}
 
+	protected void checkAccessPointAware(Object...os) {
+		if(os != null) {
+			for(Object o : os) {
+				if(o instanceof AccessPointAware) {
+					AccessPointAware apa = (AccessPointAware) o;
+					apa.setAccessPoint(this);
+				}
+			}
+		}
+	}
+	
 	private void checkInterstitialRedirect(HttpServletRequest httpRequest,
 			WaybackRequest wbRequest) 
 	throws BetterRequestException {
@@ -323,7 +341,7 @@
 		}
 	}
 	
-	private void handleReplay(WaybackRequest wbRequest, 
+	protected void handleReplay(WaybackRequest wbRequest, 
 			HttpServletRequest httpRequest, HttpServletResponse httpResponse) 
 	throws IOException, ServletException, WaybackException {
 		Resource resource = null;
@@ -399,7 +417,7 @@
 		}
 	}
 	
-	private void handleQuery(WaybackRequest wbRequest, 
+	protected void handleQuery(WaybackRequest wbRequest, 
 			HttpServletRequest httpRequest, HttpServletResponse httpResponse) 
 	throws ServletException, IOException, WaybackException {
 
@@ -909,4 +927,32 @@
 	public void setEmbargoMS(long ms) {
 		this.embargoMS = ms;
 	}
+
+	/**
+	 * @return the forceCleanQueries
+	 */
+	public boolean isForceCleanQueries() {
+		return forceCleanQueries;
+	}
+
+	/**
+	 * @param forceCleanQueries the forceCleanQueries to set
+	 */
+	public void setForceCleanQueries(boolean forceCleanQueries) {
+		this.forceCleanQueries = forceCleanQueries;
+	}
+
+	/**
+	 * @param filterFactory the filterFactory to set
+	 */
+	public void setFilterFactory(CustomResultFilterFactory filterFactory) {
+		this.filterFactory = filterFactory;
+	}
+
+	/**
+	 * @return the filterFactory
+	 */
+	public CustomResultFilterFactory getFilterFactory() {
+		return filterFactory;
+	}
 }

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Wayback-1 - Build # 39 - Still Failing:

Check console output at https://builds.archive.org:1443/job/Wayback-1/39/ to view the results.



Revision: 3512
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3512&view=rev
Author:   bradtofel
Date:     2011-09-06 03:43:17 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
INITIAL REV: new interface for AccessPoint collaboraters to be made aware of their AccessPoint

Added Paths:
-----------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/AccessPointAware.java

Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/AccessPointAware.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/AccessPointAware.java	                        (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/AccessPointAware.java	2011-09-06 03:43:17 UTC (rev 3512)
@@ -0,0 +1,34 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.wayback.webapp;
+
+/**
+ * Interface for object which need to be made aware of the AccessPoint within
+ * which they are operating
+ * 
+ * @author brad
+ *
+ */
+public interface AccessPointAware {
+	/**
+	 * @param accessPoint in use with this object
+	 */
+	public void setAccessPoint(AccessPoint accessPoint);
+}

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3511
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3511&view=rev
Author:   bradtofel
Date:     2011-09-06 03:41:31 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
LICENSE

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoPeriodTuple.java
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoResultFilterFactory.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoPeriodTuple.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoPeriodTuple.java	2011-09-06 03:40:40 UTC (rev 3510)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoPeriodTuple.java	2011-09-06 03:41:31 UTC (rev 3511)
@@ -1,3 +1,22 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
 package org.archive.wayback.webapp;
 
 public class PrefixEmbargoPeriodTuple {

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoResultFilterFactory.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoResultFilterFactory.java	2011-09-06 03:40:40 UTC (rev 3510)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoResultFilterFactory.java	2011-09-06 03:41:31 UTC (rev 3511)
@@ -1,3 +1,22 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
 package org.archive.wayback.webapp;
 
 import java.util.ArrayList;

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Wayback-1 - Build # 38 - Failure:

Check console output at https://builds.archive.org:1443/job/Wayback-1/38/ to view the results.



Revision: 3510
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3510&view=rev
Author:   bradtofel
Date:     2011-09-06 03:40:40 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
INITIAL REV: classes to simplify embargo of specific file prefixes - not very clean..

Added Paths:
-----------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/CustomResultFilterFactory.java

Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/CustomResultFilterFactory.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/CustomResultFilterFactory.java	                        (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/CustomResultFilterFactory.java	2011-09-06 03:40:40 UTC (rev 3510)
@@ -0,0 +1,37 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.wayback.webapp;
+
+import org.archive.wayback.core.CaptureSearchResult;
+import org.archive.wayback.util.ObjectFilter;
+
+/**
+ * Factory of CaptureSearchResult ObjectFilters
+ * @author brad
+ *
+ */
+public interface CustomResultFilterFactory {
+	/**
+	 * @param ap the AccessPoint is use
+	 * @return an ObjectFilter for filtering results configured for this 
+	 * AccessPoint
+	 */
+	public ObjectFilter<CaptureSearchResult> get(AccessPoint ap);
+}

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3509
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3509&view=rev
Author:   bradtofel
Date:     2011-09-06 03:38:08 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
INITIAL REV: classes to simplify embargo of specific file prefixes - not very clean..

Added Paths:
-----------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoPeriodTuple.java
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoResultFilterFactory.java

Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoPeriodTuple.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoPeriodTuple.java	                        (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoPeriodTuple.java	2011-09-06 03:38:08 UTC (rev 3509)
@@ -0,0 +1,30 @@
+package org.archive.wayback.webapp;
+
+public class PrefixEmbargoPeriodTuple {
+	protected String prefix;
+	protected long embargoMS;
+	/**
+	 * @return the prefix
+	 */
+	public String getPrefix() {
+		return prefix;
+	}
+	/**
+	 * @param prefix the prefix to set
+	 */
+	public void setPrefix(String prefix) {
+		this.prefix = prefix;
+	}
+	/**
+	 * @return the embargoMS
+	 */
+	public long getEmbargoMS() {
+		return embargoMS;
+	}
+	/**
+	 * @param embargoMS the embargoMS to set
+	 */
+	public void setEmbargoMS(long embargoMS) {
+		this.embargoMS = embargoMS;
+	}
+}

Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoResultFilterFactory.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoResultFilterFactory.java	                        (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/webapp/PrefixEmbargoResultFilterFactory.java	2011-09-06 03:38:08 UTC (rev 3509)
@@ -0,0 +1,40 @@
+package org.archive.wayback.webapp;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.archive.wayback.core.CaptureSearchResult;
+import org.archive.wayback.resourceindex.filters.CompositeFilter;
+import org.archive.wayback.resourceindex.filters.FilePrefixDateEmbargoFilter;
+import org.archive.wayback.util.ObjectFilter;
+
+public class PrefixEmbargoResultFilterFactory implements CustomResultFilterFactory {
+	protected List<PrefixEmbargoPeriodTuple> tuples = null;
+	public ObjectFilter<CaptureSearchResult> get(AccessPoint ap) {
+		if(tuples == null) {
+			return null;
+		}
+		CompositeFilter composite = new CompositeFilter();
+		ArrayList<ObjectFilter<CaptureSearchResult>> filters = 
+			new ArrayList<ObjectFilter<CaptureSearchResult>>();
+		for(PrefixEmbargoPeriodTuple tuple : tuples) {
+			filters.add(new FilePrefixDateEmbargoFilter(tuple.getPrefix(), 
+					tuple.getEmbargoMS()));
+		}
+		composite.setFilters(filters);
+		return composite;
+	}
+	/**
+	 * @return the tuples
+	 */
+	public List<PrefixEmbargoPeriodTuple> getTuples() {
+		return tuples;
+	}
+	/**
+	 * @param tuples the tuples to set
+	 */
+	public void setTuples(List<PrefixEmbargoPeriodTuple> tuples) {
+		this.tuples = tuples;
+	}
+
+}

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Wayback-1 - Build # 37 - Successful:

Check console output at https://builds.archive.org:1443/job/Wayback-1/37/ to view the results.



Access-Control - Build # 40 - Successful:

Check console output at https://builds.archive.org:1443/job/Access-Control/40/ to view the results.



Wayback-1 - Build # 36 - Successful:

Check console output at https://builds.archive.org:1443/job/Wayback-1/36/ to view the results.



Access-Control - Build # 39 - Successful:

Check console output at https://builds.archive.org:1443/job/Access-Control/39/ to view the results.



Wayback-1 - Build # 35 - Successful:

Check console output at https://builds.archive.org:1443/job/Wayback-1/35/ to view the results.



Access-Control - Build # 38 - Successful:

Check console output at https://builds.archive.org:1443/job/Access-Control/38/ to view the results.



Revision: 3508
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3508&view=rev
Author:   binzino
Date:     2011-08-29 22:05:13 +0000 (Mon, 29 Aug 2011)
Log Message:
-----------
Removed some obsolete code that was commented-out long ago.

Modified Paths:
--------------
    tags/nutchwax-0_13-JIRA-WAX-75/archive/src/java/org/archive/nutchwax/Importer.java

Modified: tags/nutchwax-0_13-JIRA-WAX-75/archive/src/java/org/archive/nutchwax/Importer.java
===================================================================
--- tags/nutchwax-0_13-JIRA-WAX-75/archive/src/java/org/archive/nutchwax/Importer.java	2011-08-18 22:52:50 UTC (rev 3507)
+++ tags/nutchwax-0_13-JIRA-WAX-75/archive/src/java/org/archive/nutchwax/Importer.java	2011-08-29 22:05:13 UTC (rev 3508)
@@ -344,8 +344,6 @@
         // We store both the normal URL and the URL+digest key for
         // later retrieval by the indexing plugin(s).
         contentMetadata.set( NutchWax.URL_KEY,            url  );
-        //contentMetadata.set( NutchWax.ORIG_KEY,           key  );
-
         contentMetadata.set( NutchWax.FILENAME_KEY,       meta.getArcFile().getName() );
         contentMetadata.set( NutchWax.FILEOFFSET_KEY,     String.valueOf( record.getHeader().getOffset( ) ) );
 
@@ -737,14 +735,12 @@
         job.setJobName( "Importer " + manifestPath );
         job.set( Nutch.SEGMENT_NAME_KEY, segmentPath.getName() );
 
-        //job.setInputPath  ( manifestPath);
         FileInputFormat.addInputPath( job, manifestPath );
         job.setInputFormat( TextInputFormat.class );
 
         job.setMapperClass ( Importer.class   );
         job.setReducerClass( Importer.class   );
 
-        //job.setOutputPath      ( segmentPath               );
         FileOutputFormat.setOutputPath( job, segmentPath );
         job.setOutputFormat    ( FetcherOutputFormat.class );
         job.setOutputKeyClass  ( Text.class                );

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3507
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3507&view=rev
Author:   binzino
Date:     2011-08-18 22:52:50 +0000 (Thu, 18 Aug 2011)

Log Message:
-----------
Fix ARI-2791.  Add magic patterns for UTF-16 HTML files.  Enable encoding detection.

Modified Paths:
--------------
    tags/nutchwax-0_13-JIRA-WAX-75/archive/src/nutch/conf/nutch-site.xml
    tags/nutchwax-0_13-JIRA-WAX-75/archive/src/nutch/conf/tika-mimetypes.xml

Modified: tags/nutchwax-0_13-JIRA-WAX-75/archive/src/nutch/conf/nutch-site.xml
===================================================================
--- tags/nutchwax-0_13-JIRA-WAX-75/archive/src/nutch/conf/nutch-site.xml	2011-08-16 11:16:32 UTC (rev 3506)
+++ tags/nutchwax-0_13-JIRA-WAX-75/archive/src/nutch/conf/nutch-site.xml	2011-08-18 22:52:50 UTC (rev 3507)
@@ -181,4 +181,12 @@
   <value>false</value>
 </property>
 
+<property>
+  <name>encodingdetector.charset.min.confidence</name>
+  <value>1</value>
+  <description>A integer between 0-100 indicating minimum confidence value
+  for charset auto-detection. Any negative value disables auto-detection.
+  </description>
+</property>
+
 </configuration>

Modified: tags/nutchwax-0_13-JIRA-WAX-75/archive/src/nutch/conf/tika-mimetypes.xml
===================================================================
--- tags/nutchwax-0_13-JIRA-WAX-75/archive/src/nutch/conf/tika-mimetypes.xml	2011-08-16 11:16:32 UTC (rev 3506)
+++ tags/nutchwax-0_13-JIRA-WAX-75/archive/src/nutch/conf/tika-mimetypes.xml	2011-08-18 22:52:50 UTC (rev 3507)
@@ -3584,6 +3584,15 @@
       <match value="&lt;H1" type="string" offset="0"/>
       <match value="&lt;!doctype HTML" type="string" offset="0"/>
       <match value="&lt;!DOCTYPE html" type="string" offset="0"/>
+      <!-- UTF-16BE BOM : Either <HTML or <html -->
+      <match value="0xfeff003c00480054004d004c" type="string" offset="0"/>
+      <match value="0xfeff003c00680074006d006c" type="string" offset="0"/>
+      <!-- UTF-16LE BOM : Either <HTML or <html -->
+      <match value="0xfffe3c00480054004d004c00" type="string" offset="0"/>
+      <match value="0xfffe3c00680074006d006c00" type="string" offset="0"/>
+      <!-- UTF-8 BOM : Either <HTML or <html -->
+      <match value="0xefbbbf3c48544d4c" type="string" offset="0"/>
+      <match value="0xefbbbf3c68746d6c" type="string" offset="0"/>
     </magic>
     <glob pattern="*.html"/>
     <glob pattern="*.htm"/>
@@ -3600,11 +3609,11 @@
       <match value="//" type="string" offset="0"/>
       <match value=";;" type="string" offset="0"/>
       <!-- UTF-16BE BOM -->
-      <match value="0xfeff" type="string" offset="0"/>
+<!--      <match value="0xfeff" type="string" offset="0"/> -->
       <!-- UTF-16LE BOM -->
-      <match value="0xfffe" type="string" offset="0"/>
+<!--      <match value="0xfffe" type="string" offset="0"/> -->
       <!-- UTF-8 BOM -->
-      <match value="0xefbbbf" type="string" offset="0"/>
+<!--      <match value="0xefbbbf" type="string" offset="0"/> -->
     </magic>
 
     <glob pattern="*.txt"/>


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.



Revision: 3506
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3506&view=rev
Author:   bradtofel
Date:     2011-08-16 11:16:32 +0000 (Tue, 16 Aug 2011)

Log Message:
-----------
1.6.2 RELEASE PREP

Modified Paths:
--------------
    branches/wayback-1_6_2/dist/pom.xml
    branches/wayback-1_6_2/pom.xml
    branches/wayback-1_6_2/wayback-core/pom.xml
    branches/wayback-1_6_2/wayback-hadoop/pom.xml
    branches/wayback-1_6_2/wayback-hadoop-java/pom.xml
    branches/wayback-1_6_2/wayback-webapp/pom.xml

Modified: branches/wayback-1_6_2/dist/pom.xml
===================================================================
--- branches/wayback-1_6_2/dist/pom.xml	2011-08-16 11:05:35 UTC (rev 3505)
+++ branches/wayback-1_6_2/dist/pom.xml	2011-08-16 11:16:32 UTC (rev 3506)
@@ -7,7 +7,7 @@
   <parent>
     <artifactId>wayback</artifactId>
     <groupId>org.archive.wayback</groupId>
-    <version>1.6.1</version>
+    <version>1.6.2</version>
   </parent>
 
   <artifactId>dist</artifactId>

Modified: branches/wayback-1_6_2/pom.xml
===================================================================
--- branches/wayback-1_6_2/pom.xml	2011-08-16 11:05:35 UTC (rev 3505)
+++ branches/wayback-1_6_2/pom.xml	2011-08-16 11:16:32 UTC (rev 3506)
@@ -7,7 +7,7 @@
   <groupId>org.archive.wayback</groupId>
   <artifactId>wayback</artifactId>
   <packaging>pom</packaging>
-  <version>1.6.1</version>
+  <version>1.6.2</version>
   <name>Wayback</name>
 
   <modules>

Modified: branches/wayback-1_6_2/wayback-core/pom.xml
===================================================================
--- branches/wayback-1_6_2/wayback-core/pom.xml	2011-08-16 11:05:35 UTC (rev 3505)
+++ branches/wayback-1_6_2/wayback-core/pom.xml	2011-08-16 11:16:32 UTC (rev 3506)
@@ -8,7 +8,7 @@
   <parent>
     <artifactId>wayback</artifactId>
     <groupId>org.archive.wayback</groupId>
-    <version>1.6.1</version>
+    <version>1.6.2</version>
   </parent>
 
   <artifactId>wayback-core</artifactId>

Modified: branches/wayback-1_6_2/wayback-hadoop/pom.xml
===================================================================
--- branches/wayback-1_6_2/wayback-hadoop/pom.xml	2011-08-16 11:05:35 UTC (rev 3505)
+++ branches/wayback-1_6_2/wayback-hadoop/pom.xml	2011-08-16 11:16:32 UTC (rev 3506)
@@ -8,7 +8,7 @@
   <parent>
     <artifactId>wayback</artifactId>
     <groupId>org.archive.wayback</groupId>
-    <version>1.6.1</version>
+    <version>1.6.2</version>
   </parent>
 
   <artifactId>wayback-hadoop</artifactId>

Modified: branches/wayback-1_6_2/wayback-hadoop-java/pom.xml
===================================================================
--- branches/wayback-1_6_2/wayback-hadoop-java/pom.xml	2011-08-16 11:05:35 UTC (rev 3505)
+++ branches/wayback-1_6_2/wayback-hadoop-java/pom.xml	2011-08-16 11:16:32 UTC (rev 3506)
@@ -8,7 +8,7 @@
   <parent>
     <artifactId>wayback</artifactId>
     <groupId>org.archive.wayback</groupId>
-    <version>1.6.1</version>
+    <version>1.6.2</version>
   </parent>
 
   <artifactId>wayback-hadoop-java</artifactId>

Modified: branches/wayback-1_6_2/wayback-webapp/pom.xml
===================================================================
--- branches/wayback-1_6_2/wayback-webapp/pom.xml	2011-08-16 11:05:35 UTC (rev 3505)
+++ branches/wayback-1_6_2/wayback-webapp/pom.xml	2011-08-16 11:16:32 UTC (rev 3506)
@@ -7,7 +7,7 @@
   <parent>
     <artifactId>wayback</artifactId>
     <groupId>org.archive.wayback</groupId>
-    <version>1.6.1</version>
+    <version>1.6.2</version>
   </parent>
 
   <artifactId>wayback-webapp</artifactId>


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.



Revision: 3505
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3505&view=rev
Author:   bradtofel
Date:     2011-08-16 11:05:35 +0000 (Tue, 16 Aug 2011)

Log Message:
-----------
BUGFIX: made automatic bouncing from a FORM query to the "clean" archival URL format optional - over a year ago, it was added, made default, and not disablable.

Modified Paths:
--------------
    branches/wayback-1_6_2/wayback-core/src/main/java/org/archive/wayback/archivalurl/requestparser/ArchivalUrlFormRequestParser.java
    branches/wayback-1_6_2/wayback-core/src/main/java/org/archive/wayback/webapp/AccessPoint.java

Modified: branches/wayback-1_6_2/wayback-core/src/main/java/org/archive/wayback/archivalurl/requestparser/ArchivalUrlFormRequestParser.java
===================================================================
--- branches/wayback-1_6_2/wayback-core/src/main/java/org/archive/wayback/archivalurl/requestparser/ArchivalUrlFormRequestParser.java	2011-08-16 10:49:16 UTC (rev 3504)
+++ branches/wayback-1_6_2/wayback-core/src/main/java/org/archive/wayback/archivalurl/requestparser/ArchivalUrlFormRequestParser.java	2011-08-16 11:05:35 UTC (rev 3505)
@@ -54,12 +54,14 @@
 				accessPoint.translateRequestPathQuery(httpRequest);
 			ArchivalUrl aUrl = new ArchivalUrl(wbRequest);
 			String bestPath = aUrl.toString();
-			if(!bestPath.equals(requestPath)) {
-				String betterURI = (wbRequest.isReplayRequest() ? 
-						accessPoint.getReplayPrefix() : 
-							accessPoint.getQueryPrefix()) 
-						+ bestPath;
-				throw new BetterRequestException(betterURI);
+			if(accessPoint.isForceCleanQueries()) {
+				if(!bestPath.equals(requestPath)) {
+					String betterURI = (wbRequest.isReplayRequest() ? 
+							accessPoint.getReplayPrefix() : 
+								accessPoint.getQueryPrefix()) 
+							+ bestPath;
+					throw new BetterRequestException(betterURI);
+				}
 			}
 		}
 		return wbRequest;

Modified: branches/wayback-1_6_2/wayback-core/src/main/java/org/archive/wayback/webapp/AccessPoint.java
===================================================================
--- branches/wayback-1_6_2/wayback-core/src/main/java/org/archive/wayback/webapp/AccessPoint.java	2011-08-16 10:49:16 UTC (rev 3504)
+++ branches/wayback-1_6_2/wayback-core/src/main/java/org/archive/wayback/webapp/AccessPoint.java	2011-08-16 11:05:35 UTC (rev 3505)
@@ -95,6 +95,7 @@
 	private boolean serveStatic = true;
 	private boolean bounceToReplayPrefix = false;
 	private boolean bounceToQueryPrefix = false;
+	private boolean forceCleanQueries = false;
 
 	private String liveWebPrefix = null;
 	private String staticPrefix = null;
@@ -848,4 +849,18 @@
 	public void setBounceToQueryPrefix(boolean bounceToQueryPrefix) {
 		this.bounceToQueryPrefix = bounceToQueryPrefix;
 	}
+
+	/**
+	 * @return the forceCleanQueries
+	 */
+	public boolean isForceCleanQueries() {
+		return forceCleanQueries;
+	}
+
+	/**
+	 * @param forceCleanQueries the forceCleanQueries to set
+	 */
+	public void setForceCleanQueries(boolean forceCleanQueries) {
+		this.forceCleanQueries = forceCleanQueries;
+	}
 }


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.



Revision: 3504
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3504&view=rev
Author:   bradtofel
Date:     2011-08-16 10:49:16 +0000 (Tue, 16 Aug 2011)

Log Message:
-----------
FEATURE: re-added list of configured AccessPoints when none match - this was accidentally removed a while back..

Modified Paths:
--------------
    branches/wayback-1_6_2/wayback-core/src/main/java/org/archive/wayback/util/webapp/PortMapper.java

Modified: branches/wayback-1_6_2/wayback-core/src/main/java/org/archive/wayback/util/webapp/PortMapper.java
===================================================================
--- branches/wayback-1_6_2/wayback-core/src/main/java/org/archive/wayback/util/webapp/PortMapper.java	2011-08-16 10:47:28 UTC (rev 3503)
+++ branches/wayback-1_6_2/wayback-core/src/main/java/org/archive/wayback/util/webapp/PortMapper.java	2011-08-16 10:49:16 UTC (rev 3504)
@@ -19,6 +19,7 @@
  */
 package org.archive.wayback.util.webapp;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.logging.Logger;
 
@@ -152,6 +153,20 @@
 		if(handler != null) {
 			return new RequestHandlerContext(handler,contextPath);
 		}
+		// Nothing matching this port:host:path. Try to help get user back on
+		// track. Note this won't help with hostname mismatches at the moment:
+		ArrayList<String> paths = new ArrayList<String>();
+		for(String tmp : pathMap.keySet()) {
+			// slice off last chunk:
+			int idx = tmp.lastIndexOf('/');
+			if(idx != -1) {
+				String path = tmp.substring(idx+1);
+				paths.add(path);
+			}
+		}
+		if(paths.size() > 0) {
+			request.setAttribute("AccessPointNames", paths);
+		}
 		return null;
 	}
 }


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




2005	Jan	Feb	Mar	Apr	May	Jun	Jul (1)	Aug (10)	Sep (36)	Oct (339)	Nov (103)	Dec (152)
2006	Jan (141)	Feb (102)	Mar (125)	Apr (203)	May (57)	Jun (30)	Jul (139)	Aug (46)	Sep (64)	Oct (105)	Nov (34)	Dec (162)
2007	Jan (81)	Feb (57)	Mar (141)	Apr (72)	May (9)	Jun (1)	Jul (144)	Aug (88)	Sep (40)	Oct (43)	Nov (34)	Dec (20)
2008	Jan (44)	Feb (45)	Mar (16)	Apr (36)	May (8)	Jun (77)	Jul (177)	Aug (66)	Sep (8)	Oct (33)	Nov (13)	Dec (37)
2009	Jan (2)	Feb (5)	Mar (8)	Apr	May (36)	Jun (19)	Jul (46)	Aug (8)	Sep (1)	Oct (66)	Nov (61)	Dec (10)
2010	Jan (13)	Feb (16)	Mar (38)	Apr (76)	May (47)	Jun (32)	Jul (35)	Aug (45)	Sep (20)	Oct (61)	Nov (24)	Dec (16)
2011	Jan (22)	Feb (34)	Mar (11)	Apr (8)	May (24)	Jun (23)	Jul (11)	Aug (42)	Sep (81)	Oct (48)	Nov (21)	Dec (20)
2012	Jan (30)	Feb (25)	Mar (4)	Apr (6)	May (1)	Jun (5)	Jul (5)	Aug (8)	Sep (6)	Oct (6)	Nov	Dec

archive-access-cvs Mailing List for Web Archive Access Utilities (Page 11)

archive-access-cvs — CVS commits