[Smartfrog-checkins] SF.net SVN: smartfrog:[7728] trunk/core/extras/citerank

[<st...@us...>]

Revision: 7728
          http://smartfrog.svn.sourceforge.net/smartfrog/?rev=7728&view=rev
Author:   steve_l
Date:     2009-06-25 16:30:31 +0000 (Thu, 25 Jun 2009)

Log Message:
-----------
SFOS-1242 	 add the counters

Modified Paths:
--------------
    trunk/core/extras/citerank/build.xml
    trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRank.java
    trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRankTool.java

Modified: trunk/core/extras/citerank/build.xml
===================================================================
--- trunk/core/extras/citerank/build.xml	2009-06-25 16:20:43 UTC (rev 7727)
+++ trunk/core/extras/citerank/build.xml	2009-06-25 16:30:31 UTC (rev 7728)
@@ -177,10 +177,10 @@
       <arg value="/playground/citeseer"/>
       <arg value="100"/>
       <arg value="0.0000001"/>
-      <propertyset >
+      <syspropertyset  >
         <propertyref prefix="test."/>
         <propertyref prefix="runtime."/>
-      </propertyset>
+      </syspropertyset>
 
     </java>
   </target>

Modified: trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRank.java
===================================================================
--- trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRank.java	2009-06-25 16:20:43 UTC (rev 7727)
+++ trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRank.java	2009-06-25 16:30:31 UTC (rev 7728)
@@ -24,6 +24,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.mapred.Counters;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -78,6 +79,10 @@
         String output = outpath + File.separator + CiteRankTool.CURRENT_RANKS;
         int iterationLimit = Integer.parseInt(args[2]);
         final double toleranceArg = Double.parseDouble(args[3]);
+        
+        //reset the counters. This is an abuse of a singleton and should be replaced
+        //with instance values as soon as possible
+        resetCounters();
         //clean the data up
         exec("Data cleanup", new CheckingData(), inpath, output);
         //count the data
@@ -136,10 +141,16 @@
             exec("HTMLTable", new HTMLTable(),
                     outpath, Integer.toString(HTML_TABLE_ROWS), Integer.toString(iterations));
         }
+
+        Counters totals = getCounters();
+        totals.log(LOG);
         return 0;
     }
 
     public static void main(String[] args) throws Exception {
         System.exit(ToolRunner.run(new Configuration(), new CiteRank(), args));
     }
+    
+    
+    
 }

Modified: trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRankTool.java
===================================================================
--- trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRankTool.java	2009-06-25 16:20:43 UTC (rev 7727)
+++ trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRankTool.java	2009-06-25 16:30:31 UTC (rev 7728)
@@ -28,6 +28,8 @@
 import org.apache.hadoop.mapred.FileOutputFormat;
 import org.apache.hadoop.mapred.JobClient;
 import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hadoop.mapred.Counters;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
@@ -77,6 +79,9 @@
     public static final String OPTION_PIXELS_ONLY = "pixels.only";
     public static final String OPTION_REPORT_CITESEER_URL = "report.citeseer.url";
 
+    private static Counters counters;
+    
+    
     /**
      * Get the name of this tool
      *
@@ -119,8 +124,26 @@
         return -1;
     }
 
+    
+    public static synchronized void resetCounters() {
+        counters = null;
+    } 
+    
+    public static synchronized void addCounters(Counters values) {
+        if (counters==null) {
+            counters = values;
+        } else {
+            counters.incrAllCounters(values);
+        }
+    }
+    
+    public static synchronized Counters getCounters() {
+        return counters;
+    }
+    
     protected static int runJob(JobConf conf) throws IOException {
-        JobClient.runJob(conf);
+        RunningJob job = JobClient.runJob(conf);
+        addCounters(job.getCounters());
         return 0;
     }
 
@@ -157,4 +180,26 @@
         LOG.info("Running " + description);
         ToolRunner.run(getConf(), instance, args);
     }
+    /*
+         [java] 09/06/25 16:47:03 INFO mapred.JobClient: Job complete: job_local_0002
+     [java] 09/06/25 16:47:03 INFO mapred.JobClient: Counters: 11
+     [java] 09/06/25 16:47:03 INFO mapred.JobClient:   File Systems
+     [java] 09/06/25 16:47:03 INFO mapred.JobClient:     Local bytes read=423569384
+     [java] 09/06/25 16:47:03 INFO mapred.JobClient:     Local bytes written=390279157
+     [java] 09/06/25 16:47:03 INFO mapred.JobClient:   Map-Reduce Framework
+     [java] 09/06/25 16:47:03 INFO mapred.JobClient:     Reduce input groups=1
+     [java] 09/06/25 16:47:03 INFO mapred.JobClient:     Combine output records=4
+     [java] 09/06/25 16:47:03 INFO mapred.JobClient:     Map input records=717172
+     [java] 09/06/25 16:47:03 INFO mapred.JobClient:     Reduce output records=1
+     [java] 09/06/25 16:47:03 INFO mapred.JobClient:     Map output bytes=10040408
+     [java] 09/06/25 16:47:03 INFO mapred.JobClient:     Map input bytes=17258927
+     [java] 09/06/25 16:47:03 INFO mapred.JobClient:     Combine input records=717175
+     [java] 09/06/25 16:47:03 INFO mapred.JobClient:     Map output records=717172
+     [java] 09/06/25 16:47:03 INFO mapred.JobClient:     Reduce input records=1
+     [java] 09/06/25 16:47:03 INFO citerank.CiteRankTool: Running InitializeRanks
+
+     */
+    
+    
+    
 }


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.