[Archive-access-cvs] SF.net SVN: archive-access:[3213] tags/nutchwax-0_13-JIRA-WAX-75/archive/ src/java/org/archive/nutchwax/IndexerMapReduce.java

[<bi...@us...>]

Revision: 3213
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3213&view=rev
Author:   binzino
Date:     2010-08-09 23:52:32 +0000 (Mon, 09 Aug 2010)

Log Message:
-----------
Added date merging hacks.

Modified Paths:
--------------
    tags/nutchwax-0_13-JIRA-WAX-75/archive/src/java/org/archive/nutchwax/IndexerMapReduce.java

Modified: tags/nutchwax-0_13-JIRA-WAX-75/archive/src/java/org/archive/nutchwax/IndexerMapReduce.java
===================================================================
--- tags/nutchwax-0_13-JIRA-WAX-75/archive/src/java/org/archive/nutchwax/IndexerMapReduce.java	2010-08-09 23:52:07 UTC (rev 3212)
+++ tags/nutchwax-0_13-JIRA-WAX-75/archive/src/java/org/archive/nutchwax/IndexerMapReduce.java	2010-08-09 23:52:32 UTC (rev 3213)
@@ -73,7 +73,15 @@
       final Writable value = values.next().get(); // unwrap
 
       if (value instanceof ParseData) {
-        parseData = (ParseData)value;
+        if ( parseData != null )
+          {
+            // HACK: Merge dates
+            parseData = mergeDates( key, parseData, (ParseData) value );
+          }
+        else
+          {
+            parseData = (ParseData)value;
+          }
       } else if (value instanceof ParseText) {
         parseText = (ParseText)value;
       } else if (LOG.isWarnEnabled()) {
@@ -118,6 +126,25 @@
     output.collect(key, doc);
   }
 
+  private ParseData mergeDates( Text key, ParseData d, ParseData s )
+  {
+    Metadata dest = d.getContentMeta();
+    Metadata src  = s.getContentMeta();
+
+    if ( dest == null ) return s;
+    if ( src  == null ) return d;
+    
+    // HACK: All we care about right now are the dates.
+    String[] sourceDates = src.getValues( "date" );
+    for ( String date : sourceDates )
+      {
+        LOG.warn( "Merging: " + key + " : " + date );
+        dest.add( "date", date );
+      }
+
+    return d;
+  }
+
   public void close() throws IOException { }
 
   public static void initMRJob(Collection<Path> segments,


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.