From: <st...@us...> - 2009-06-22 18:41:12
|
Revision: 7707 http://smartfrog.svn.sourceforge.net/smartfrog/?rev=7707&view=rev Author: steve_l Date: 2009-06-22 18:40:11 +0000 (Mon, 22 Jun 2009) Log Message: ----------- SFOS-1242 add a citerank project to stress test and benchmark hadoop clusters Modified Paths: -------------- trunk/core/components/hadoop/doc/dynamic_hadoop_clusters.ppt trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/dfs/DfsListDirImpl.java trunk/core/extras/citerank/build.xml trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRank.java trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRankTool.java trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/bindings.sf trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop.sf Added Paths: ----------- trunk/core/extras/citerank/conf/dundee/ trunk/core/extras/citerank/conf/dundee/cluster.properties trunk/core/extras/citerank/conf/dundee/hadoop-site.xml Modified: trunk/core/components/hadoop/doc/dynamic_hadoop_clusters.ppt =================================================================== (Binary files differ) Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/dfs/DfsListDirImpl.java =================================================================== --- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/dfs/DfsListDirImpl.java 2009-06-21 22:19:20 UTC (rev 7706) +++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/dfs/DfsListDirImpl.java 2009-06-22 18:40:11 UTC (rev 7707) @@ -76,6 +76,7 @@ throw new SmartFrogLivenessException("Path not found in the remote filesystem: " + path, this); } StringBuilder builder = new StringBuilder(); + builder.append("Listing of ").append(path); for (FileStatus file : stats) { size += file.getLen(); builder.append(file.getPath().getName()); Modified: trunk/core/extras/citerank/build.xml =================================================================== --- trunk/core/extras/citerank/build.xml 2009-06-21 22:19:20 UTC (rev 7706) +++ trunk/core/extras/citerank/build.xml 2009-06-22 18:40:11 UTC (rev 7707) @@ -109,7 +109,7 @@ </deploy> </target> - <target name="upload" depends="ready-to-deploy" + <target name="upload2" depends="ready-to-deploy" description="copy the datasets to the target cluster"> <!-- timeout="${timeout}" --> <sf-run-locally classpathref="tests.run.classpath"> @@ -122,7 +122,7 @@ </target> <!--variant that does the upload via a local daemon--> - <target name="upload2" depends="ready-to-deploy" + <target name="upload" depends="ready-to-deploy" description="copy the datasets to the target cluster"> <deploy host="localhost"> <application name="upload"> Copied: trunk/core/extras/citerank/conf/dundee/cluster.properties (from rev 7706, trunk/core/extras/citerank/conf/annecy/cluster.properties) =================================================================== --- trunk/core/extras/citerank/conf/dundee/cluster.properties (rev 0) +++ trunk/core/extras/citerank/conf/dundee/cluster.properties 2009-06-22 18:40:11 UTC (rev 7707) @@ -0,0 +1,6 @@ +# fsck http://192.168.174.130:8021/fsck +# pages http://192.168.174.130:8021/listPaths/ +# http://192.168.174.130:8021/listPaths/playground/citeseer +# http://192.168.174.130:8021/listPaths/datasets/citeseer +fs.default.name=hdfs://192.168.174.130:8020/ +mapred.job.tracker=ipc://192.168.174.130:8012/ Copied: trunk/core/extras/citerank/conf/dundee/hadoop-site.xml (from rev 7706, trunk/core/extras/citerank/conf/annecy/hadoop-site.xml) =================================================================== --- trunk/core/extras/citerank/conf/dundee/hadoop-site.xml (rev 0) +++ trunk/core/extras/citerank/conf/dundee/hadoop-site.xml 2009-06-22 18:40:11 UTC (rev 7707) @@ -0,0 +1,15 @@ +<?xml version="1.0"?> + +<configuration> + + <property> + <name>fs.default.name</name> + <value>hdfs://192.168.174.130:8020/</value> + </property> + + <property> + <name>mapred.job.tracker</name> + <value>ipc://192.168.174.130:8012/</value> + </property> + +</configuration> \ No newline at end of file Modified: trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRank.java =================================================================== --- trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRank.java 2009-06-21 22:19:20 UTC (rev 7706) +++ trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRank.java 2009-06-22 18:40:11 UTC (rev 7707) @@ -109,7 +109,7 @@ } if (HTML_TABLE) { - exec("SortRanks", new SortRanks(), input, sortedRanksDir); + exec("SortRanks #"+(iterations+1), new SortRanks(), input, sortedRanksDir); Path htmlsource = new Path(sortedRanksDir + File.separator + "part-00000"); BufferedReader in = null; PrintWriter out = null; @@ -133,7 +133,8 @@ exec("SortRanks", new SortRanks(), input, sortedRanksDir); if (HTML_TABLE) { - exec("HTMLTable", new HTMLTable(), outpath, Integer.toString(HTML_TABLE_ROWS), Integer.toString(iterations)); + exec("HTMLTable", new HTMLTable(), + outpath, Integer.toString(HTML_TABLE_ROWS), Integer.toString(iterations)); } return 0; } Modified: trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRankTool.java =================================================================== --- trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRankTool.java 2009-06-21 22:19:20 UTC (rev 7706) +++ trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/CiteRankTool.java 2009-06-22 18:40:11 UTC (rev 7707) @@ -152,7 +152,7 @@ * @throws Exception anything that went wrong */ protected void exec(String description, CiteRankTool instance, String... args) throws Exception { - LOG.info("Running "+description); + LOG.info("Running " + description); ToolRunner.run(getConf(), instance, args); } } Modified: trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/bindings.sf =================================================================== --- trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/bindings.sf 2009-06-21 22:19:20 UTC (rev 7706) +++ trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/bindings.sf 2009-06-22 18:40:11 UTC (rev 7707) @@ -24,7 +24,9 @@ WORKER_THREADS 8; +CLUSTER_LIVENESS_DELAY 0 ; + namenode.host PROPERTY test.namenode; jobtracker.host PROPERTY test.jobtracker; datanode1.host PROPERTY test.datanode1; Modified: trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop.sf =================================================================== --- trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop.sf 2009-06-21 22:19:20 UTC (rev 7706) +++ trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop.sf 2009-06-22 18:40:11 UTC (rev 7707) @@ -101,7 +101,9 @@ HadoopClusterSingleProcess extends HadoopFilesystemSingleProcess { + sfLivenessDelay CLUSTER_LIVENESS_DELAY; + jobtracker extends ClusterJobTrackerCompound { fs.default.name fs.name; } @@ -121,7 +123,8 @@ tasktracker:sfProcessName TASKTRACKER1_PROCESS; } -HadoopCluster extends HadoopClusterSingleProcess; +HadoopCluster extends HadoopClusterSingleProcess { +} ForkedCluster extends HadoopClusterMultiProcess; This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |