From: <st...@us...> - 2009-07-02 23:06:30
|
Revision: 7772 http://smartfrog.svn.sourceforge.net/smartfrog/?rev=7772&view=rev Author: steve_l Date: 2009-07-02 23:06:27 +0000 (Thu, 02 Jul 2009) Log Message: ----------- SFOS-905 Move from direct inherited configuration to a "Cluster" CD that defines the cluster Modified Paths: -------------- trunk/core/extras/citerank/build.xml trunk/core/extras/citerank/conf/home/cluster.properties trunk/core/extras/citerank/conf/home/hadoop-site.xml trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/components.sf trunk/core/extras/citerank/test/org/smartfrog/services/hadoop/benchmark/test/system/components.sf trunk/core/extras/citerank/test/org/smartfrog/services/hadoop/benchmark/test/unit/UtilsTest.java trunk/core/extras/hadoop-cluster/build.xml trunk/core/extras/hadoop-cluster/ivy.xml trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop.sf trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/mapreduce/components.sf Added Paths: ----------- trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/cluster4.sf trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop-clusterconfig.sf trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop-inlineconfig.sf trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/ trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/FullClusterJobSubmissionTest.java trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/components.sf trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/testJobSubmission.sf Modified: trunk/core/extras/citerank/build.xml =================================================================== --- trunk/core/extras/citerank/build.xml 2009-07-02 22:59:03 UTC (rev 7771) +++ trunk/core/extras/citerank/build.xml 2009-07-02 23:06:27 UTC (rev 7772) @@ -34,7 +34,7 @@ <!-- override point --> <property file="build.properties"/> - <!--unit tests are on--> + <!-- tests are on--> <property name="system.tests" value="true"/> <property name="junit.timeout" value="120000000"/> @@ -187,6 +187,19 @@ </java> </target> + <target name="submit2file" depends="ready-to-deploy" + description="export the cluster resource to an expanded file"> + <property name="build.sf.dir" location="${build.dir}/sf"/> + <mkdir dir="${build.sf.dir}"/> + <property name="build.default.sf" location="${build.sf.dir}/default.sf"/> + <sf-expandfully + resource="/${default.sf}" + dest="${build.default.sf}" + classpathRef="tests.run.classpath" + /> + <echo>file is ${build.default.sf}</echo> + </target> + </project> \ No newline at end of file Modified: trunk/core/extras/citerank/conf/home/cluster.properties =================================================================== --- trunk/core/extras/citerank/conf/home/cluster.properties 2009-07-02 22:59:03 UTC (rev 7771) +++ trunk/core/extras/citerank/conf/home/cluster.properties 2009-07-02 23:06:27 UTC (rev 7772) @@ -1,2 +1,3 @@ -fs.default.name=hdfs://192.168.1.2:8020/ -mapred.job.tracker=ipc://192.168.1.2:8012/ +ipaddress=192.168.1.88 +fs.default.name=hdfs://${ipaddress}:8020/ +mapred.job.tracker=ipc://${ipaddress}:8012/ Modified: trunk/core/extras/citerank/conf/home/hadoop-site.xml =================================================================== --- trunk/core/extras/citerank/conf/home/hadoop-site.xml 2009-07-02 22:59:03 UTC (rev 7771) +++ trunk/core/extras/citerank/conf/home/hadoop-site.xml 2009-07-02 23:06:27 UTC (rev 7772) @@ -4,12 +4,12 @@ <property> <name>fs.default.name</name> - <value>hdfs://192.168.1.2:8020/</value> + <value>hdfs://192.168.1.88:8020/</value> </property> <property> <name>mapred.job.tracker</name> - <value>ipc://192.168.1.2:8012/</value> + <value>ipc://192.168.1.88:8012/</value> </property> </configuration> \ No newline at end of file Modified: trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/components.sf =================================================================== --- trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/components.sf 2009-07-02 22:59:03 UTC (rev 7771) +++ trunk/core/extras/citerank/src/org/smartfrog/services/hadoop/benchmark/citerank/components.sf 2009-07-02 23:06:27 UTC (rev 7772) @@ -81,15 +81,15 @@ - UploadTestData extends Sequence { - upload extends UploadCiteseerData { - cluster LAZY PARENT:ATTRIB testcluster; - } +UploadTestData extends Sequence { + upload extends UploadCiteseerData { + cluster LAZY PARENT:ATTRIB testcluster; + } - ls extends lsCiteseerDir { - cluster LAZY PARENT:ATTRIB testcluster - } + ls extends lsCiteseerDir { + cluster LAZY PARENT:ATTRIB testcluster } +} Upload extends ClusterActions { @@ -151,7 +151,4 @@ } - - - } \ No newline at end of file Modified: trunk/core/extras/citerank/test/org/smartfrog/services/hadoop/benchmark/test/system/components.sf =================================================================== --- trunk/core/extras/citerank/test/org/smartfrog/services/hadoop/benchmark/test/system/components.sf 2009-07-02 22:59:03 UTC (rev 7771) +++ trunk/core/extras/citerank/test/org/smartfrog/services/hadoop/benchmark/test/system/components.sf 2009-07-02 23:06:27 UTC (rev 7772) @@ -0,0 +1,19 @@ +/* (C) Copyright 2009 Hewlett-Packard Development Company, LP + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +For more information: www.smartfrog.org + +*/ Modified: trunk/core/extras/citerank/test/org/smartfrog/services/hadoop/benchmark/test/unit/UtilsTest.java =================================================================== --- trunk/core/extras/citerank/test/org/smartfrog/services/hadoop/benchmark/test/unit/UtilsTest.java 2009-07-02 22:59:03 UTC (rev 7771) +++ trunk/core/extras/citerank/test/org/smartfrog/services/hadoop/benchmark/test/unit/UtilsTest.java 2009-07-02 23:06:27 UTC (rev 7772) @@ -1,3 +1,22 @@ +/* (C) Copyright 2009 Hewlett-Packard Development Company, LP + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +For more information: www.smartfrog.org + +*/ package org.smartfrog.services.hadoop.benchmark.test.unit; import junit.framework.TestCase; Modified: trunk/core/extras/hadoop-cluster/build.xml =================================================================== --- trunk/core/extras/hadoop-cluster/build.xml 2009-07-02 22:59:03 UTC (rev 7771) +++ trunk/core/extras/hadoop-cluster/build.xml 2009-07-02 23:06:27 UTC (rev 7772) @@ -54,12 +54,8 @@ <property name="ivy.enabled" value="true"/> <!-- override point --> <property file="build.properties"/> - <!--System tests are only turned on if Java6 or later is in use--> - <!-- - <condition property="system.tests" > - <available classname="java.net.CookieManager"/> - </condition> - --> + <!-- tests are on--> + <property name="system.tests" value="true"/> <property name="junit.timeout" value="120000000"/> <!--time for a single test to execute--> <property name="test.timeout.execute" value="120000"/> @@ -84,7 +80,9 @@ <property name="cluster" value="org/smartfrog/extras/hadoop/cluster" /> <property name="cluster.sf" value="${cluster}/cluster.sf" /> <property name="cluster2.sf" value="${cluster}/cluster2.sf" /> - <property name="default.sf" value="${cluster}/cluster3.sf" /> + <property name="cluster3.sf" value="${cluster}/cluster3.sf" /> + <property name="cluster4.sf" value="${cluster}/cluster4.sf" /> + <property name="default.sf" value="${cluster3.sf}" /> <property name="deploy.sf" value="${cluster.sf}" /> <!--this sets the name for the tests--> <property name="deploy.host" value="${test.namenode}" /> @@ -98,6 +96,13 @@ deploy.sf=${deploy.sf} </echo> + <property name="build.sf.dir" location="${build.dir}/sf" /> + <mkdir dir="${build.sf.dir}"/> + + <property name="build.default.sf" location="${build.sf.dir}/default.sf" /> + <property name="src.scripts.dir" location="src/org/smartfrog/extras/hadoop/cluster/services/scripts" /> + <property name="envVariables.sh" location="${src.scripts.dir}/setSFEnvVariables.sh" /> + </target> @@ -115,6 +120,7 @@ test.jobtracker=${test.jobtracker} test.datanode1=${test.datanode1} test.tasktracker1=${test.tasktracker1} + daemon.jvmargs ="${daemon.jvmargs}" </echo> </target> @@ -137,6 +143,15 @@ </deploy> </target> + <target name="cluster4" depends="ready-to-deploy" + description="deploy a full cluster"> + <deploy> + <application name="cluster4" + descriptor="/org/smartfrog/extras/hadoop/cluster/cluster4.sf"/> + </deploy> + </target> + + <target name="filesystem" depends="ready-to-deploy" description="deploy a filesystem"> <deploy> @@ -211,13 +226,7 @@ <target name="cluster2file" depends="ready-to-deploy" description="export the cluster resource to an expanded file"> - <property name="build.sf.dir" location="${build.dir}/sf" /> - <mkdir dir="${build.sf.dir}"/> - <property name="build.default.sf" location="${build.sf.dir}/default.sf" /> - <property name="src.scripts.dir" location="src/org/smartfrog/extras/hadoop/cluster/services/scripts" /> - <property name="envVariables.sh" location="${src.scripts.dir}/setSFEnvVariables.sh" /> - <sf-expandfully resource="/${default.sf}" dest="${build.default.sf}" @@ -226,6 +235,17 @@ <echo> file is ${build.default.sf}</echo> </target> + <target name="cluster4file" depends="ready-to-deploy" + description="export the cluster resource to an expanded file"> + + <sf-expandfully + resource="/${default.sf}" + dest="${build.default.sf}" + classpathRef="tests.run.classpath" + /> + <echo> file is ${build.default.sf}</echo> + </target> + <!-- this target inits/prints --> <target name="init-ssh" depends="ready-to-deploy"> <property name="ssh.host" value="localhost" /> Modified: trunk/core/extras/hadoop-cluster/ivy.xml =================================================================== --- trunk/core/extras/hadoop-cluster/ivy.xml 2009-07-02 22:59:03 UTC (rev 7771) +++ trunk/core/extras/hadoop-cluster/ivy.xml 2009-07-02 23:06:27 UTC (rev 7772) @@ -74,7 +74,7 @@ <dependency org="org.smartfrog" name="sf-anubis" changing="true" rev="latest.integration" - conf="server->default"/> + conf="redist->default"/> <dependency org="org.smartfrog" name="sf-hadoop" Copied: trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/cluster4.sf (from rev 7767, trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/cluster.sf) =================================================================== --- trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/cluster4.sf (rev 0) +++ trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/cluster4.sf 2009-07-02 23:06:27 UTC (rev 7772) @@ -0,0 +1,24 @@ +/* (C) Copyright 2009 Hewlett-Packard Development Company, LP + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +For more information: www.smartfrog.org + +*/ +#include "/org/smartfrog/extras/hadoop/cluster/components.sf" + + + +sfConfig extends FullClusterWithBindings; \ No newline at end of file Copied: trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop-clusterconfig.sf (from rev 7753, trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop.sf) =================================================================== --- trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop-clusterconfig.sf (rev 0) +++ trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop-clusterconfig.sf 2009-07-02 23:06:27 UTC (rev 7772) @@ -0,0 +1,75 @@ +/* (C) Copyright 2009 Hewlett-Packard Development Company, LP + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +For more information: www.smartfrog.org + +*/ + + + + + + +/** + Configures a cluster from cluster references + + This system is designed to split based on a hostname flag; it looks at the name + and decides whether or not to be in charge + + */ + + + + +FullCluster extends ConfiguredCluster { + + threads WORKER_THREADS; + max_tasks 6; + sfLivenessDelay CLUSTER_LIVENESS_DELAY; + fs.default.name TBD; + + namenodeHostname TBD; + jobtrackerHostname namenodeHostname; + filesystemURL ("hdfs://" ++ (namenodeHostname) ++ ":" ++ NameNodeIpcPort ++ "/"); + fs.default.name filesystemURL; + + namenode:checkRunning false; + + + cluster:fs.default.name filesystemURL; + cluster:dfs.datanode.handler.count threads; + cluster:mapred.job.tracker.handler.count threads; + cluster:mapred.tasktracker.map.tasks.maximum max_tasks; + + localhost extends Localhost; + +} + +FullClusterWithBindings extends FullCluster { + namenodeHostname namenode.host; + jobtrackerHostname jobtracker.host; +} + +/* +FullClusterMultiProcess extends FullClusterWithBindings { + namenode:sfProcessName NAMENODE_PROCESS; + datanode:sfProcessName DATANODE1_PROCESS; + jobtracker:sfProcessName JOBTRACKER_PROCESS; + tasktracker:sfProcessName TASKTRACKER1_PROCESS; +} + +*/ + Copied: trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop-inlineconfig.sf (from rev 7753, trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop.sf) =================================================================== --- trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop-inlineconfig.sf (rev 0) +++ trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop-inlineconfig.sf 2009-07-02 23:06:27 UTC (rev 7772) @@ -0,0 +1,138 @@ +/* (C) Copyright 2009 Hewlett-Packard Development Company, LP + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +For more information: www.smartfrog.org + +*/ + + + +/** + Configures a cluster from inline options + */ + +ClusterNameNodeCompound extends TransientNameNodeCompound { + + namenodeHost extends Localhost; + + threads WORKER_THREADS; + //localhost.name APPLY lookupLocalhost; + namenodeHostname LAZY namenodeHost:hostname; + hostname namenodeHostname; + fs.default.name ("hdfs://" ++ (hostname) ++ ":" ++ NameNodeIpcPort ++ "/"); + namenode:dfs.datanode.handler.count threads; + namenode:checkRunning false; +} + +ClusterDataNodeCompound extends TransientDataNodeCompound { + threads WORKER_THREADS; + fs.default.name TBD; + workerHost extends Localhost; + hostname HadoopConfiguration:ALL_ZEROS; +} + +ClusterJobTrackerCompound extends TransientJobTrackerCompound { + threads WORKER_THREADS; + fs.default.name TBD; + + jobtrackerHost extends Localhost; + jobtracker:mapred.job.tracker.handler.count threads; +} + + +ClusterTaskTrackerCompound extends TransientTaskTrackerCompound { + threads WORKER_THREADS; + tasktracker.http.threads threads; + workerHost extends Localhost; +} + + +/** + * The parallel container is used here to leave up any part of the system + * which can come up. This allows us to permit partial cluster failure, without + * failing ourselves. + */ +HadoopClusterCompound extends Compound { + + +} + +HadoopFilesystemSingleProcess extends HadoopClusterCompound { + + fs.name LAZY namenode:namenode:fs.default.name; + + namenode extends ClusterNameNodeCompound { + } + + datanode extends ClusterDataNodeCompound { + fs.default.name fs.name; + } + + +} + +HadoopFilesystemMultiProcess extends HadoopFilesystemSingleProcess { + namenode:sfProcessName NAMENODE_PROCESS; + datanode:sfProcessName DATANODE1_PROCESS; +} + +HadoopFilesystem extends HadoopClusterSingleProcess { + +} + + + +HadoopClusterSingleProcess extends HadoopFilesystemSingleProcess { + + sfLivenessDelay CLUSTER_LIVENESS_DELAY; + + + jobtracker extends ClusterJobTrackerCompound { + fs.default.name fs.name; + } + + + tasktracker extends ClusterTaskTrackerCompound { + fs.default.name fs.name; + tasktracker:mapred.tasktracker.map.tasks.maximum 6; + } + +} + +HadoopClusterMultiProcess extends HadoopClusterSingleProcess { + namenode:sfProcessName NAMENODE_PROCESS; + datanode:sfProcessName DATANODE1_PROCESS; + jobtracker:sfProcessName JOBTRACKER_PROCESS; + tasktracker:sfProcessName TASKTRACKER1_PROCESS; +} + +HadoopCluster extends HadoopClusterSingleProcess { +} + +ForkedCluster extends HadoopClusterMultiProcess; + +DistributedFilesystem extends HadoopFilesystem { + namenode:sfProcessHost namenode.host; + datanode:sfProcessHost datanode1.host; +} + +/*DistributedCluster extends HadoopClusterSingleProcess { + namenode:sfProcessHost namenode.host; + datanode:sfProcessHost datanode1.host; + jobtracker:sfProcessHost jobtracker.host; + tasktracker:sfProcessHost tasktracker1.host; +}*/ + Modified: trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop.sf =================================================================== --- trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop.sf 2009-07-02 22:59:03 UTC (rev 7771) +++ trunk/core/extras/hadoop-cluster/src/org/smartfrog/extras/hadoop/cluster/hadoop.sf 2009-07-02 23:06:27 UTC (rev 7772) @@ -17,119 +17,7 @@ For more information: www.smartfrog.org */ - #include "/org/smartfrog/extras/hadoop/cluster/bindings.sf" - - -ClusterNameNodeCompound extends TransientNameNodeCompound { - - namenodeHost extends Localhost; - - threads WORKER_THREADS; - //localhost.name APPLY lookupLocalhost; - namenodeHostname LAZY namenodeHost:hostname; - hostname namenodeHostname; - fs.default.name ("hdfs://" ++ (hostname) ++ ":" ++ NameNodeIpcPort ++ "/"); - namenode:dfs.datanode.handler.count threads; - namenode:checkRunning false; -} - -ClusterDataNodeCompound extends TransientDataNodeCompound { - threads WORKER_THREADS; - fs.default.name TBD; - workerHost extends Localhost; - hostname HadoopConfiguration:ALL_ZEROS; -} - -ClusterJobTrackerCompound extends TransientJobTrackerCompound { - threads WORKER_THREADS; - fs.default.name TBD; - - jobtrackerHost extends Localhost; - jobtracker:mapred.job.tracker.handler.count threads; -} - - -ClusterTaskTrackerCompound extends TransientTaskTrackerCompound { - threads WORKER_THREADS; - tasktracker.http.threads threads; - workerHost extends Localhost; -} - - -/** - * The parallel container is used here to leave up any part of the system - * which can come up. This allows us to permit partial cluster failure, without - * failing ourselves. - */ -HadoopClusterCompound extends Compound { - - -} - -HadoopFilesystemSingleProcess extends HadoopClusterCompound { - - fs.name LAZY namenode:namenode:fs.default.name; - - namenode extends ClusterNameNodeCompound { - } - - datanode extends ClusterDataNodeCompound { - fs.default.name fs.name; - } - - -} - -HadoopFilesystemMultiProcess extends HadoopFilesystemSingleProcess { - namenode:sfProcessName NAMENODE_PROCESS; - datanode:sfProcessName DATANODE1_PROCESS; -} - -HadoopFilesystem extends HadoopFilesystemSingleProcess { - -} - - - -HadoopClusterSingleProcess extends HadoopFilesystemSingleProcess { - - sfLivenessDelay CLUSTER_LIVENESS_DELAY; - - - jobtracker extends ClusterJobTrackerCompound { - fs.default.name fs.name; - } - - - tasktracker extends ClusterTaskTrackerCompound { - fs.default.name fs.name; - tasktracker:mapred.tasktracker.map.tasks.maximum 6; - } - -} - -HadoopClusterMultiProcess extends HadoopClusterSingleProcess { - namenode:sfProcessName NAMENODE_PROCESS; - datanode:sfProcessName DATANODE1_PROCESS; - jobtracker:sfProcessName JOBTRACKER_PROCESS; - tasktracker:sfProcessName TASKTRACKER1_PROCESS; -} - -HadoopCluster extends HadoopClusterSingleProcess { -} - -ForkedCluster extends HadoopClusterMultiProcess; - -DistributedFilesystem extends HadoopFilesystem { - namenode:sfProcessHost namenode.host; - datanode:sfProcessHost datanode1.host; -} - -DistributedCluster extends HadoopCluster { - namenode:sfProcessHost namenode.host; - datanode:sfProcessHost datanode1.host; - jobtracker:sfProcessHost jobtracker.host; - tasktracker:sfProcessHost tasktracker1.host; -} - +#include "/org/smartfrog/services/hadoop/examples/cluster-driven.sf" +#include "/org/smartfrog/extras/hadoop/cluster/hadoop-inlineconfig.sf" +#include "/org/smartfrog/extras/hadoop/cluster/hadoop-clusterconfig.sf" Copied: trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/FullClusterJobSubmissionTest.java (from rev 7767, trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/mapreduce/JobSubmissionTest.java) =================================================================== --- trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/FullClusterJobSubmissionTest.java (rev 0) +++ trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/FullClusterJobSubmissionTest.java 2009-07-02 23:06:27 UTC (rev 7772) @@ -0,0 +1,43 @@ +/* (C) Copyright 2009 Hewlett-Packard Development Company, LP + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +For more information: www.smartfrog.org + +*/ +package org.smartfrog.extras.hadoop.cluster.test.system.fullcluster; + +import org.smartfrog.extras.hadoop.cluster.test.HadoopTestBase; + +/** + * Created 05-Jan-2009 16:12:47 + */ + +public class FullClusterJobSubmissionTest extends HadoopTestBase { + + String PACKAGE = + "/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/"; + + public FullClusterJobSubmissionTest(String name) { + super(name); + } + + public void testJobSubmission() throws Throwable { + checkMapRedCluster(); + expectSuccessfulTestRunOrSkip(PACKAGE, "testJobSubmission.sf"); + enableFailOnPortCheck(); + } + +} \ No newline at end of file Copied: trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/components.sf (from rev 7767, trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/mapreduce/components.sf) =================================================================== --- trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/components.sf (rev 0) +++ trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/components.sf 2009-07-02 23:06:27 UTC (rev 7772) @@ -0,0 +1,42 @@ +/* (C) Copyright 2009 Hewlett-Packard Development Company, LP + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +For more information: www.smartfrog.org + +*/ +#include "/org/smartfrog/extras/hadoop/cluster/test/system/components.sf" + +testFullCluster extends ExpectLive { + action FullClusterWithBindings; + + +} + + + +testJobSubmission extends testFullCluster { + description "Submit a job to a cluster and copy out the result"; + + tests extends MapReduceSequence { + + LocalDataDir PROPERTY test.work.dir; + jobtracker LAZY PARENT:action:jobtracker:service; + namenode LAZY PARENT:action:namenode:service; + datanode LAZY PARENT:action:datanode:service; + + } + +} \ No newline at end of file Copied: trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/testJobSubmission.sf (from rev 7767, trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/mapreduce/testJobSubmission.sf) =================================================================== --- trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/testJobSubmission.sf (rev 0) +++ trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/testJobSubmission.sf 2009-07-02 23:06:27 UTC (rev 7772) @@ -0,0 +1,23 @@ +/* (C) Copyright 2009 Hewlett-Packard Development Company, LP + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +For more information: www.smartfrog.org + +*/ + +#include "/org/smartfrog/extras/hadoop/cluster/test/system/fullcluster/components.sf" + +sfConfig extends testJobSubmission; \ No newline at end of file Modified: trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/mapreduce/components.sf =================================================================== --- trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/mapreduce/components.sf 2009-07-02 22:59:03 UTC (rev 7771) +++ trunk/core/extras/hadoop-cluster/test/org/smartfrog/extras/hadoop/cluster/test/system/mapreduce/components.sf 2009-07-02 23:06:27 UTC (rev 7772) @@ -26,7 +26,6 @@ description "Submit a job to a cluster and copy out the result"; tests extends MapReduceSequence { - //waitForJobTracker:timeout 60000 ; LocalDataDir PROPERTY test.work.dir; jobtracker LAZY PARENT:action:jobtracker:jobtracker; This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |