Menu
▾
▴
archive-access-cvs
Revision: 3210
http://archive-access.svn.sourceforge.net/archive-access/?rev=3210&view=rev
Author: binzino
Date: 2010-08-09 16:04:53 +0000 (Mon, 09 Aug 2010)
Log Message:
-----------
Initial revision.
Added Paths:
-----------
tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/DateIndexer.java
Added: tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/DateIndexer.java
===================================================================
--- tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/DateIndexer.java (rev 0)
+++ tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/DateIndexer.java 2010-08-09 16:04:53 UTC (rev 3210)
@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2008 Internet Archive.
+ *
+ * This file is part of the archive-access tools project
+ * (http://sourceforge.net/projects/archive-access).
+ *
+ * The archive-access tools are free software; you can redistribute them and/or
+ * modify them under the terms of the GNU Lesser Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or any
+ * later version.
+ *
+ * The archive-access tools are distributed in the hope that they will be
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser Public License along with
+ * the archive-access tools; if not, write to the Free Software Foundation,
+ * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package org.archive.nutchwax.index;
+
+import java.util.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.Inlinks;
+import org.apache.nutch.indexer.IndexingException;
+import org.apache.nutch.indexer.IndexingFilter;
+import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.indexer.lucene.LuceneWriter;
+import org.apache.nutch.metadata.Metadata;
+import org.apache.nutch.parse.Parse;
+
+/**
+ *
+ */
+public class DateIndexer implements IndexingFilter
+{
+ public static final Log LOG = LogFactory.getLog( DateIndexer.class );
+
+ private Configuration conf;
+
+ public void setConf( Configuration conf )
+ {
+ this.conf = conf;
+ }
+
+ public Configuration getConf()
+ {
+ return this.conf;
+ }
+
+ /**
+ * <p>Set Lucene document field to fixed value.</p>
+ * <p>
+ * Remove field if specified value is <code>null</code>.
+ * </p>
+ */
+ public NutchDocument filter( NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks )
+ throws IndexingException
+ {
+ Metadata meta = parse.getData().getContentMeta();
+
+ Set<String> dates = new HashSet<String>( Arrays.asList( meta.getValues( "date" ) ) );
+
+ for ( String date : dates )
+ {
+ doc.add( "date", date );
+ doc.add( "year", date.substring( 0, 4 ) );
+ doc.add( "yearmonth", date.substring( 0, 6 ) );
+ }
+
+ return doc;
+ }
+
+ public void addIndexBackendOptions( Configuration conf )
+ {
+ LuceneWriter.addFieldOptions( "date", LuceneWriter.STORE.YES, LuceneWriter.INDEX.NO, conf );
+ LuceneWriter.addFieldOptions( "year", LuceneWriter.STORE.NO, LuceneWriter.INDEX.UNTOKENIZED, conf );
+ LuceneWriter.addFieldOptions( "yearmonth", LuceneWriter.STORE.NO, LuceneWriter.INDEX.UNTOKENIZED, conf );
+ }
+
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
Revision: 3211
http://archive-access.svn.sourceforge.net/archive-access/?rev=3211&view=rev
Author: binzino
Date: 2010-08-09 23:51:25 +0000 (Mon, 09 Aug 2010)
Log Message:
-----------
Added some comments.
Modified Paths:
--------------
tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/DateIndexer.java
Modified: tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/DateIndexer.java
===================================================================
--- tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/DateIndexer.java 2010-08-09 16:04:53 UTC (rev 3210)
+++ tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/DateIndexer.java 2010-08-09 23:51:25 UTC (rev 3211)
@@ -36,7 +36,16 @@
import org.apache.nutch.parse.Parse;
/**
+ * Add fields to NutchDocument related to dates.
*
+ * All of the crawl dates are taken from the Parse object, reduced to
+ * a set of unique dates (in case of redundancies), then the dates
+ * are added to the NutchDocument in multiple fields:
+ * <ul>
+ * <li><code>date</code> stored for later retrieval.</li>
+ * <li><code>year</code> only the year (YYYY), indexed not stored.</l>
+ * <li><code>yearmonth</code> year and month (YYYYMM), indexed not stored.</li>
+ * </ul>
*/
public class DateIndexer implements IndexingFilter
{
@@ -55,16 +64,14 @@
}
/**
- * <p>Set Lucene document field to fixed value.</p>
- * <p>
- * Remove field if specified value is <code>null</code>.
- * </p>
+ *
*/
public NutchDocument filter( NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks )
throws IndexingException
{
Metadata meta = parse.getData().getContentMeta();
+ // Use a Set to reduce list of dates to just unique values.
Set<String> dates = new HashSet<String>( Arrays.asList( meta.getValues( "date" ) ) );
for ( String date : dates )
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
Revision: 3306
http://archive-access.svn.sourceforge.net/archive-access/?rev=3306&view=rev
Author: binzino
Date: 2010-10-27 07:00:57 +0000 (Wed, 27 Oct 2010)
Log Message:
-----------
Changed to just store the date, no indexing.
Modified Paths:
--------------
tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/DateIndexer.java
Modified: tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/DateIndexer.java
===================================================================
--- tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/DateIndexer.java 2010-10-27 06:56:42 UTC (rev 3305)
+++ tags/nutchwax-0_13-JIRA-WAX-75/archive/src/plugin/index-nutchwax/src/java/org/archive/nutchwax/index/DateIndexer.java 2010-10-27 07:00:57 UTC (rev 3306)
@@ -77,8 +77,6 @@
for ( String date : dates )
{
doc.add( "date", date );
- doc.add( "year", date.substring( 0, 4 ) );
- doc.add( "yearmonth", date.substring( 0, 6 ) );
}
return doc;
@@ -87,8 +85,6 @@
public void addIndexBackendOptions( Configuration conf )
{
LuceneWriter.addFieldOptions( "date", LuceneWriter.STORE.YES, LuceneWriter.INDEX.NO, conf );
- LuceneWriter.addFieldOptions( "year", LuceneWriter.STORE.NO, LuceneWriter.INDEX.UNTOKENIZED, conf );
- LuceneWriter.addFieldOptions( "yearmonth", LuceneWriter.STORE.NO, LuceneWriter.INDEX.UNTOKENIZED, conf );
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
×
Want the latest updates on software, tech news, and AI?
Get latest updates about software, tech news, and AI from SourceForge directly in your inbox once a month.
Thanks for helping keep SourceForge clean.
X