Menu
▾
▴
[Smartfrog-checkins] SF.net SVN: smartfrog:[7252] trunk/core
|
From: <st...@us...> - 2009-01-09 16:54:59
|
Revision: 7252
http://smartfrog.svn.sourceforge.net/smartfrog/?rev=7252&view=rev
Author: steve_l
Date: 2009-01-09 16:54:04 +0000 (Fri, 09 Jan 2009)
Log Message:
-----------
SFOS-1040 Create a hadoop-cluster component that includes everything needed to run hadoop on a real or virtual cluster
Modified Paths:
--------------
trunk/core/extras/hadoop-cluster/build.xml
trunk/core/extras/hadoop-cluster/ivy.xml
trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/cluster.sf
trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop.sf
trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/ClusterTest.java
trunk/core/release/metadata/servers/rpm.centos.properties
Added Paths:
-----------
trunk/core/extras/hadoop-cluster/build.properties
Added: trunk/core/extras/hadoop-cluster/build.properties
===================================================================
--- trunk/core/extras/hadoop-cluster/build.properties (rev 0)
+++ trunk/core/extras/hadoop-cluster/build.properties 2009-01-09 16:54:04 UTC (rev 7252)
@@ -0,0 +1,6 @@
+#vmware.image=192.168.174.130
+vmware.image=127.0.0.1
+namenode=${vmware.image}
+datanode=${vmware.image}
+tasktracker=${vmware.image}
+jobtracker=${vmware.image}
\ No newline at end of file
Modified: trunk/core/extras/hadoop-cluster/build.xml
===================================================================
--- trunk/core/extras/hadoop-cluster/build.xml 2009-01-08 18:04:17 UTC (rev 7251)
+++ trunk/core/extras/hadoop-cluster/build.xml 2009-01-09 16:54:04 UTC (rev 7252)
@@ -60,4 +60,53 @@
</target>
+ <!-- ========================================================== -->
+ <!-- deploy; requires a host -->
+ <!-- ========================================================== -->
+ <target name="ready-to-deploy" depends="common.ready-to-deploy"
+ description="deploy a namenode">
+
+
+ <echo level="verbose">tests.run.classpath=${toString:tests.run.classpath}</echo>
+ <presetdef name="hadoop" >
+ <sf-deploy classpathref="tests.run.classpath"
+ logStackTraces="true"
+ host="${namenode}"
+ timeout="30000"
+ >
+ <codebase url="${codebase}"/>
+ <syspropertyset>
+ <propertyref prefix="test."/>
+ </syspropertyset>
+ </sf-deploy>
+ </presetdef>
+ </target>
+
+ <!-- ========================================================== -->
+ <!-- deploy; requires a host -->
+ <!-- ========================================================== -->
+ <target name="cluster" depends="ready-to-deploy"
+ description="deploy a cluster">
+ <hadoop>
+ <application name="cluster"
+ descriptor="/${cluster.sf}"/>
+ </hadoop>
+ </target>
+
+ <target name="namenode" depends="ready-to-deploy"
+ description="deploy a namenode">
+ <hadoop>
+ <application name="namenode"
+ descriptor="/org/smartfrog/extras/hadoop/cluster/services/namenode/default.sf"/>
+ </hadoop>
+ </target>
+
+ <target name="datanode" depends="ready-to-deploy"
+ description="deploy a namenode">
+ <hadoop>
+ <application name="datanode"
+ descriptor="/org/smartfrog/extras/hadoop/cluster/services/datanode/default.sf"/>
+ </hadoop>
+ </target>
+
</project>
\ No newline at end of file
Modified: trunk/core/extras/hadoop-cluster/ivy.xml
===================================================================
--- trunk/core/extras/hadoop-cluster/ivy.xml 2009-01-08 18:04:17 UTC (rev 7251)
+++ trunk/core/extras/hadoop-cluster/ivy.xml 2009-01-09 16:54:04 UTC (rev 7252)
@@ -63,12 +63,15 @@
name="sf-anubis" changing="true"
rev="latest.integration"
conf="runtime->default"/>
-
<dependency org="org.smartfrog"
+ name="sf-csvfiles" changing="true"
+ rev="latest.integration"
+ conf="runtime->default"/>
+ <dependency org="org.smartfrog"
name="sf-hadoop"
changing="true"
rev="latest.integration"
- conf="runtime->server"/>
+ conf="runtime->server,default"/>
<dependency org="org.smartfrog"
name="sf-rpmtools" changing="true"
Modified: trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/cluster.sf
===================================================================
--- trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/cluster.sf 2009-01-08 18:04:17 UTC (rev 7251)
+++ trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/cluster.sf 2009-01-09 16:54:04 UTC (rev 7252)
@@ -30,11 +30,10 @@
}
-/*
datanode extends ClusterDataNodeCompound {
}
- */
+
/*
jobTracker extends ClusterJobTrackerCompound {
@@ -49,3 +48,5 @@
*/
}
+
+sfConfig extends HadoopCluster;
\ No newline at end of file
Modified: trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop.sf
===================================================================
--- trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop.sf 2009-01-08 18:04:17 UTC (rev 7251)
+++ trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop.sf 2009-01-09 16:54:04 UTC (rev 7252)
@@ -38,21 +38,33 @@
}
-ClusterDataNodeCompound extends TransientDataNodeCompound, Bindings {
- jobtracker:mapred.job.tracker.handler.count threads;
-}
-ClusterNameNodeCompound extends TransientDataNodeCompound, Bindings {
+ClusterNameNodeCompound extends TransientNameNodeCompound{
+ threads WORKER_THREADS;
+ fs.default.name namenode.filesystem;
namenode:dfs.datanode.handler.count threads;
+ namenode:checkRunning false;
}
-ClusterJobTrackerCompound extends TransientJobTrackerCompound, Bindings {
+ClusterDataNodeCompound extends TransientDataNodeCompound {
+ threads WORKER_THREADS;
+ fs.default.name namenode.filesystem;
+ //datanode:mapred.job.tracker.handler.count threads;
+}
+/*
+
+ClusterJobTrackerCompound extends TransientJobTrackerCompound {
+ threads WORKER_THREADS;
+ fs.default.name namenode.filesystem;
jobtracker:mapred.job.tracker.handler.count threads;
}
-ClusterTaskTrackerCompound extends TransientJobTrackerCompound, Bindings {
+ClusterTaskTrackerCompound extends TransientJobTrackerCompound {
+ threads WORKER_THREADS;
+ fs.default.name namenode.filesystem;
tasktracker.http.threads threads;
}
+*/
Modified: trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/ClusterTest.java
===================================================================
--- trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/ClusterTest.java 2009-01-08 18:04:17 UTC (rev 7251)
+++ trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/ClusterTest.java 2009-01-09 16:54:04 UTC (rev 7252)
@@ -19,9 +19,15 @@
*/
package org.smartfrog.extras.hadoop.cluster.test.system;
+import org.smartfrog.test.DeployingTestBase;
+
/**
* Created 05-Jan-2009 16:12:47
*/
-public class ClusterTest {
+public class ClusterTest extends DeployingTestBase {
+
+ public ClusterTest(String name) {
+ super(name);
+ }
}
Modified: trunk/core/release/metadata/servers/rpm.centos.properties
===================================================================
--- trunk/core/release/metadata/servers/rpm.centos.properties 2009-01-08 18:04:17 UTC (rev 7251)
+++ trunk/core/release/metadata/servers/rpm.centos.properties 2009-01-09 16:54:04 UTC (rev 7252)
@@ -1,7 +1,7 @@
#This is an example rpm upload target to a nearby vmware image, one with the appropriate ssh keys for root
#and the nominated user
-rpm.ssh.server=192.168.1.4
+rpm.ssh.server=192.168.174.130
rpm.ssh.user=smartfrog
rpm.ssh.keyfile=${user.home}/.ssh/id_rsa
rpm.ssh.passphrase=
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|