[Archive-access-cvs] SF.net SVN: archive-access:[3571] trunk/archive-access/projects/ia-tools/ src/main/java/org/archive/hadoop/mapreduce

[<bra...@us...>]

Revision: 3571
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3571&view=rev
Author:   bradtofel
Date:     2011-11-19 00:57:07 +0000 (Sat, 19 Nov 2011)
Log Message:
-----------
Using old ZipNumRecordWriter - new one has a bug 

Modified Paths:
--------------
    trunk/archive-access/projects/ia-tools/src/main/java/org/archive/hadoop/mapreduce/ZipNumOutputFormat.java
    trunk/archive-access/projects/ia-tools/src/main/java/org/archive/hadoop/mapreduce/ZipNumRecordWriter.java

Modified: trunk/archive-access/projects/ia-tools/src/main/java/org/archive/hadoop/mapreduce/ZipNumOutputFormat.java
===================================================================
--- trunk/archive-access/projects/ia-tools/src/main/java/org/archive/hadoop/mapreduce/ZipNumOutputFormat.java	2011-11-19 00:50:24 UTC (rev 3570)
+++ trunk/archive-access/projects/ia-tools/src/main/java/org/archive/hadoop/mapreduce/ZipNumOutputFormat.java	2011-11-19 00:57:07 UTC (rev 3571)
@@ -51,7 +51,10 @@
 		FSDataOutputStream mainOut = mainFs.create(mainFile, false);
 		FSDataOutputStream summaryOut = summaryFs.create(summaryFile, false);
 		if(dayLimit == -1) {
-			return new ZipNumRecordWriter(count, mainOut, summaryOut);
+			// This (should be) a better implementation, but appears to have a 
+			// bug - summary files are empty in some cases.. Should track it down
+//			return new ZipNumRecordWriter(count, mainOut, summaryOut);
+			return new ZipNumRecordWriterOld(count, mainOut, summaryOut);
 		} else {
 			return new OvercrawlZipNumRecordWriter(count,dayLimit, mainOut, summaryOut);
 		}

Modified: trunk/archive-access/projects/ia-tools/src/main/java/org/archive/hadoop/mapreduce/ZipNumRecordWriter.java
===================================================================
--- trunk/archive-access/projects/ia-tools/src/main/java/org/archive/hadoop/mapreduce/ZipNumRecordWriter.java	2011-11-19 00:50:24 UTC (rev 3570)
+++ trunk/archive-access/projects/ia-tools/src/main/java/org/archive/hadoop/mapreduce/ZipNumRecordWriter.java	2011-11-19 00:57:07 UTC (rev 3571)
@@ -9,6 +9,12 @@
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.archive.format.gzip.zipnum.ZipNumWriter;
 
+/**
+ * Warning - this has a bug.. leaves empty SUMMARY files in some cases.
+ * 
+ * @author brad
+ *
+ */
 public class ZipNumRecordWriter extends RecordWriter<Text, Text>{
 	protected ZipNumWriter znw;
 	

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.