archive-access-cvs Mailing List for Web Archive Access Utilities (Page 21)

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 422-6466

Revision: 3305
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3305&view=rev
Author:   binzino
Date:     2010-10-27 06:56:42 +0000 (Wed, 27 Oct 2010)

Log Message:
-----------
Initial revision.

Added Paths:
-----------
    tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/TypeFilter.java
    tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/TypeNormalizer.java
    tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/URLFilter.java

Added: tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/TypeFilter.java
===================================================================

--- tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/TypeFilter.java	                        (rev 0)
+++ tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/TypeFilter.java	2010-10-27 06:56:42 UTC (rev 3305)
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.archive.nutchwax.index;
+
+import java.io.*;
+import java.util.*;
+
+public class TypeFilter 
+{
+  public static final String[] DEFAULT_ALLOWED = 
+    {
+      "text/html", 
+      "text/plain",
+      "application/pdf", 
+      // MS Office document types
+      "application/msword", 
+      "application/vnd.ms-powerpoint", 
+      // OpenOffice document types
+      "application/vnd.oasis.opendocument.text",
+      "application/vnd.oasis.opendocument.presentation",
+      "application/vnd.oasis.opendocument.spreadsheet",
+    };
+
+  private Set<String>    allowed;
+  private TypeNormalizer normalizer;
+
+  public TypeFilter( )
+  {
+    
+  }
+
+  public TypeFilter( Set<String> allowed, TypeNormalizer normalizer )
+  {
+    this.allowed    = allowed;
+    this.normalizer = normalizer;
+  }
+
+  public void setTypeNormalizer( TypeNormalizer normalizer )
+  {
+    this.normalizer = normalizer;
+  }
+
+  public void setAllowed( Set<String> allowed )
+  {
+    this.allowed = allowed;
+  }
+
+  public Set<String> getAllowed( )
+  {
+    return this.allowed;
+  }
+  
+  public boolean isAllowed( String type )
+  {
+    // If no explicit list of allowed types, allow them all.
+    if ( this.allowed == null || this.allowed.size( ) == 0 )
+      {
+        return true;
+      }
+
+    // De-alias it.
+    type = this.normalizer.normalize( type );
+
+    return allowed.contains( type );
+  }
+  
+  public static Set<String> parse( String s )
+  {
+    Set<String> types = new HashSet<String>( );
+
+    for ( String type : s.split( "\\s+" ) )
+      {
+        if ( type.length() < 1 ) continue ;
+
+        types.add( type );
+      }
+
+    return types;
+  }
+
+  public static Set<String> getDefaultAllowed( )
+  {
+    Set<String> defaults = new HashSet<String>( );
+    
+    for ( String allowed : DEFAULT_ALLOWED )
+      {
+        defaults.add( allowed );
+      }
+
+    return defaults;
+  }
+
+}

Added: tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/TypeNormalizer.java
===================================================================
--- tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/TypeNormalizer.java	                        (rev 0)
+++ tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/TypeNormalizer.java	2010-10-27 06:56:42 UTC (rev 3305)
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.archive.nutchwax.index;
+
+import java.io.*;
+import java.util.*;
+
+public class TypeNormalizer
+{
+  // Maps alias->canonical
+  public static final String[][] DEFAULT_ALIASES =
+    {
+      // PDF aliases
+      { "application/x-pdf", "application/pdf" },
+      // HTML aliases.
+      { "application/xhtml+xml", "text/html" },
+      // MS Word aliases.
+      { "application/vnd.ms-word", "application/msword" },
+      { "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/msword" },
+      // PowerPoint aliases.
+      {"application/mspowerpoint",     "application/vnd.ms-powerpoint" },
+      {"application/ms-powerpoint",    "application/vnd.ms-powerpoint" },
+      {"application/mspowerpnt",       "application/vnd.ms-powerpoint" },
+      {"application/vnd-mspowerpoint", "application/vnd.ms-powerpoint" },
+      {"application/powerpoint",       "application/vnd.ms-powerpoint" },
+      {"application/x-powerpoint",     "application/vnd.ms-powerpoint" },
+      {"application/vnd.openxmlformats-officedocument.presentationml.presentation", "application/vnd.ms-powerpoint" },
+    };
+
+  private Map<String,String> aliases;
+
+  public static Map<String,String> getDefaultAliases( )
+  {
+    Map<String,String> defaults = new HashMap<String,String>( );
+
+    for ( String[] alias : DEFAULT_ALIASES )
+      {
+        defaults.put( alias[0], alias[1] );
+      }
+
+    return defaults;
+  }
+
+  public static Map<String,String> parseAliases( String s )
+  {
+    Map<String,String> aliases = new HashMap<String,String>( );
+    
+    for ( String line : s.split( "\\s+" ) )
+      {
+        if ( line.length() < 1 ) continue ;
+
+        String[] tokens = line.split( "[:,]" );
+        
+        if ( tokens.length < 2 ) continue ;
+        
+        String type = tokens[0];
+        
+        if ( type.length() < 1 ) continue ;
+
+        for ( int i = 1; i < tokens.length ; i++ )
+          {
+            aliases.put( tokens[i], type );
+          }
+      }
+
+    return aliases;
+  }
+
+  public void setAliases( Map<String,String> aliases )
+  {
+    this.aliases = aliases;
+  }
+  
+  public Map<String,String> getAliases( )
+  {
+    return this.aliases;
+  }
+  
+  public String normalize( String type )
+  {
+    // Chop off anything after a ';' character.  This is
+    // for stuff like: "text/html; charset=utf-8"
+    int p = type.indexOf( ';' );
+    if ( p >= 0 ) type = type.substring( 0, p ).trim();
+
+    if ( this.aliases != null && this.aliases.containsKey( type ) )
+      {
+        type = this.aliases.get( type );
+      }
+
+    return type;
+  }
+
+}

Added: tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/URLFilter.java
===================================================================
--- tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/URLFilter.java	                        (rev 0)
+++ tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/URLFilter.java	2010-10-27 06:56:42 UTC (rev 3305)
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.archive.nutchwax.index;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
+public class URLFilter
+{
+  public static final String[] DEFAULT_PROHIBITED = 
+    {
+      "/robots.txt",
+      "/favicon.ico",
+    };
+
+  private Set<String> prohibited;
+
+  public URLFilter( Set<String> prohibited )
+  {
+    this.prohibited = prohibited;
+  }
+
+  public void setProhibited( Set<String> prohibited )
+  {
+    this.prohibited = prohibited;
+  }
+
+  public Set<String> getProhibited( )
+  {
+    return this.prohibited;
+  }
+  
+  public boolean isAllowed( URI uri )
+  {
+    String path = uri.getRawPath( );
+
+    return ! this.prohibited.contains( path );
+  }
+  
+  public static Set<String> parse( String s )
+  {
+    Set<String> paths = new HashSet<String>( );
+
+    for ( String type : s.split( "\\s+" ) )
+      {
+        if ( type.length() < 1 ) continue ;
+
+        paths.add( type );
+      }
+
+    return paths;
+  }
+
+  public static Set<String> getDefaultProhibited( )
+  {
+    Set<String> defaults = new HashSet<String>( );
+    
+    for ( String prohibited : DEFAULT_PROHIBITED )
+      {
+        defaults.add( prohibited );
+      }
+
+    return defaults;
+  }
+
+}


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




2005	Jan	Feb	Mar	Apr	May	Jun	Jul (1)	Aug (10)	Sep (36)	Oct (339)	Nov (103)	Dec (152)
2006	Jan (141)	Feb (102)	Mar (125)	Apr (203)	May (57)	Jun (30)	Jul (139)	Aug (46)	Sep (64)	Oct (105)	Nov (34)	Dec (162)
2007	Jan (81)	Feb (57)	Mar (141)	Apr (72)	May (9)	Jun (1)	Jul (144)	Aug (88)	Sep (40)	Oct (43)	Nov (34)	Dec (20)
2008	Jan (44)	Feb (45)	Mar (16)	Apr (36)	May (8)	Jun (77)	Jul (177)	Aug (66)	Sep (8)	Oct (33)	Nov (13)	Dec (37)
2009	Jan (2)	Feb (5)	Mar (8)	Apr	May (36)	Jun (19)	Jul (46)	Aug (8)	Sep (1)	Oct (66)	Nov (61)	Dec (10)
2010	Jan (13)	Feb (16)	Mar (38)	Apr (76)	May (47)	Jun (32)	Jul (35)	Aug (45)	Sep (20)	Oct (61)	Nov (24)	Dec (16)
2011	Jan (22)	Feb (34)	Mar (11)	Apr (8)	May (24)	Jun (23)	Jul (11)	Aug (42)	Sep (81)	Oct (48)	Nov (21)	Dec (20)
2012	Jan (30)	Feb (25)	Mar (4)	Apr (6)	May (1)	Jun (5)	Jul (5)	Aug (8)	Sep (6)	Oct (6)	Nov	Dec

archive-access-cvs Mailing List for Web Archive Access Utilities (Page 21)

archive-access-cvs — CVS commits