Menu
▾
▴
[Smartfrog-checkins] SF.net SVN: smartfrog:[8248] trunk/core/components/cloudfarmer
|
From: <st...@us...> - 2010-03-09 17:59:35
|
Revision: 8248
http://smartfrog.svn.sourceforge.net/smartfrog/?rev=8248&view=rev
Author: steve_l
Date: 2010-03-09 17:58:41 +0000 (Tue, 09 Mar 2010)
Log Message:
-----------
SFOS-1442 more hadoop support
Modified Paths:
--------------
trunk/core/components/cloudfarmer/build.xml
trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/cluster_add_dynamic.jsp
trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/cluster_list.jsp
trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/cluster_list_in_role.jsp
trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/create_role_instance.jsp
trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/clusters/ClusterAllocationHandler.java
trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/model/cluster/ClusterController.java
trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/model/cluster/RoleAllocationRequest.java
trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/servlets/HostsAndRolesServlet.java
trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/common/ClusterFarmerUtils.java
trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/deployment/NodeDeploymentOverSSH.java
Modified: trunk/core/components/cloudfarmer/build.xml
===================================================================
--- trunk/core/components/cloudfarmer/build.xml 2010-03-08 17:25:47 UTC (rev 8247)
+++ trunk/core/components/cloudfarmer/build.xml 2010-03-09 17:58:41 UTC (rev 8248)
@@ -66,7 +66,7 @@
<property name="test.mombasa.port" value="8081" />
<property name="test.mombasa.url" value="http://${test.mombasa.host}:${test.mombasa.port}/" />
<property name="test.hadoop-site.xml" value="${test.mombasa.url}hadoop-site.xml" />
- <property name="test.cluster.properties" value="${test.mombasa.url}cluster.properties" />
+ <property name="test.cluster.properties" value="${test.mombasa.url}hosts.properties?prefix=test.cluster" />
<!-- <property name="deploy.sf" value="${mombasa.sf}" />-->
@@ -170,8 +170,10 @@
<property name="hadoop.config.dir" location="${build.dir}/hadoop-config" />
<mkdir dir="${hadoop.config.dir}" />
<property name="hadoop.site.xml" location="${hadoop.config.dir}/hadoop-site.xml" />
- <property name="cluster.properties" location="${hadoop.config.dir}/cluster.properties?prefix=test.cluster" />
-
+ <property name="cluster.properties" location="${hadoop.config.dir}/cluster.properties" />
+ <property name="remote.dir" value="/root" />
+ <property name="remote.settings.dir" value="${remote.dir}/settings" />
+ <property name="init-classpath" value='source setSFProperties; export CLASSPATH="${remote.settings.dir}:$CLASSPATH"'/>
</target>
<target name="GET" depends="ready-to-get"
@@ -195,13 +197,53 @@
dest="${cluster.properties}" />
<property file="${cluster.properties}" />
</target>
-
- <target name="terasort" depends="cluster.properties" >
+
+ <target name="cluster-bind" depends="cluster.properties, hadoop-site.xml" >
<fail unless="test.cluster.role.master.1" >
- No cluster master found: has anything been deployed
+ No cluster master found: has anything been deployed?
</fail>
<property name="cluster.master" value="${test.cluster.role.master.1}" />
- <echo > working with ${test.cluster.role.master.1}</echo>
+ <echo > working with ${cluster.master}</echo>
+ </target>
+
+ <target name="cluster-upload-settings" depends="cluster-bind, hadoop-site.xml" >
+ <sshexec host="${cluster.master}"
+ username="root"
+ trust="true"
+ password="${test.cluster.password}"
+ command="rm -rf ${remote.settings.dir} ${remote.dir}/in ${remote.dir}/out; mkdir ${remote.settings.dir}"/>
+ <!--verbose="true"-->
+ <scp file="${hadoop.site.xml}"
+ todir="root@${cluster.master}:${remote.settings.dir}"
+ trust="true"
+ password="${test.cluster.password}"
+ />
+ </target>
+
+ <target name="terasort" depends="cluster-upload-settings" >
+ <property name="terasort.size" value="10000000" />
+ <echo>
+Now:
+ssh root@${cluster.master}
+
+${init-classpath}
+rm -rf in out
+time java org.apache.hadoop.util.RunJar $SFHOME/lib/hadoop-mapred-examples-*.jar teragen ${terasort.size} in -verbose
+time java -Xmx1024m org.apache.hadoop.util.RunJar $SFHOME/lib/hadoop-mapred-examples-*.jar terasort in out -verbose
+ </echo>
+ </target>
+
+
+ <target name="job-queue" depends="cluster-bind">
+ <property name="jobqueue" value="org.apache.hadoop.mapred.JobQueueClient"/>
+ <property name="jobqueue.commands" value="-list -showjobs -verbose"/>
+ <sshexec host="${cluster.master}"
+ username="root"
+ trust="true"
+ password="${test.cluster.password}"
+ command='${init-classpath}; java ${jobqueue} ${jobqueue.commands}'/>
+ </target>
+
<!--
ssh ${cluster.master}
@@ -216,7 +258,7 @@
source setSFProperties
- export CLASSPATH="/home/root/settings:$CLASSPATH"
+ export CLASSPATH="/root/settings:$CLASSPATH"
echo $CLASSPATH
the main class in the examples file runs the ExampleDriver which takes the example name as param #1,
@@ -234,12 +276,11 @@
<property name="terasort.size" value="100000000" />
rm -rf in
- java org.apache.hadoop.util.RunJar $SFHOME/lib/hadoop-mapred-examples-*.jar teragen 100000000 in
+ java org.apache.hadoop.util.RunJar $SFHOME/lib/hadoop-mapred-examples-*.jar teragen 100000000 in -verbose
rm -rf out
time java -Xmx1024m org.apache.hadoop.util.RunJar $SFHOME/lib/hadoop-mapred-examples-*.jar terasort in out
-->
- </target>
</project>
\ No newline at end of file
Modified: trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/cluster_add_dynamic.jsp
===================================================================
--- trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/cluster_add_dynamic.jsp 2010-03-08 17:25:47 UTC (rev 8247)
+++ trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/cluster_add_dynamic.jsp 2010-03-09 17:58:41 UTC (rev 8248)
@@ -23,7 +23,7 @@
<%@ include file="/html/mombasa-portlet/cluster/init.jsp" %>
-<p>Add Hosts </p>
+<p>Add hosts to form a Hadoop cluster; every machine will become a datanode and job tracker.</p>
<html:form action="/mombasa-portlet/cluster/add_dynamic/process" method="post" focus="minWorkers">
Modified: trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/cluster_list.jsp
===================================================================
--- trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/cluster_list.jsp 2010-03-08 17:25:47 UTC (rev 8247)
+++ trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/cluster_list.jsp 2010-03-09 17:58:41 UTC (rev 8248)
@@ -36,6 +36,8 @@
<th class="padding"></th>
<th>Role</th>
<th class="padding"></th>
+ <th>State</th>
+ <th class="padding"></th>
<th>Application</th>
<th class="padding"></th>
<th>Description</th>
@@ -61,7 +63,6 @@
<td class="padding"></td>
<td>
<logic:notEmpty name="host" property="role">
-
<html:link action="/mombasa-portlet/cluster/listInRole"
paramId="role" paramName="host" paramProperty="role">
<bean:write name="host" property="role"/>
@@ -69,6 +70,8 @@
</logic:notEmpty>
</td>
<td class="padding"></td>
+ <td><bean:write name="host" property="state"/></td>
+ <td class="padding"></td>
<td><bean:write name="host" property="application"/></td>
<td class="padding"></td>
<td>
Modified: trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/cluster_list_in_role.jsp
===================================================================
--- trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/cluster_list_in_role.jsp 2010-03-08 17:25:47 UTC (rev 8247)
+++ trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/cluster_list_in_role.jsp 2010-03-09 17:58:41 UTC (rev 8248)
@@ -34,6 +34,8 @@
<tr class="header">
<th>Hostname</th>
<th class="padding"></th>
+ <th>State</th>
+ <th class="padding"></th>
<th>Application</th>
<th class="padding"></th>
<th>Role</th>
@@ -52,6 +54,8 @@
</html:link>
</td>
<td class="padding"></td>
+ <td><bean:write name="host" property="state"/></td>
+ <td class="padding"></td>
<td><bean:write name="host" property="application"/></td>
<td class="padding"></td>
<td>
Modified: trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/create_role_instance.jsp
===================================================================
--- trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/create_role_instance.jsp 2010-03-08 17:25:47 UTC (rev 8247)
+++ trunk/core/components/cloudfarmer/site/html/mombasa-portlet/cluster/create_role_instance.jsp 2010-03-09 17:58:41 UTC (rev 8248)
@@ -54,12 +54,12 @@
</td>
<td class="padding"></td>
<td class="field">
- <html:text name="clusterCreateRoleInstanceForm" property="rolename" styleId="role"/>
+ <html:text name="clusterCreateRoleInstanceForm" property="role" styleId="role"/>
</td>
</tr>
<%--
- <html:hidden name="clusterCreateRoleInstanceForm" property="rolename" value="${role}"/>
+ <html:hidden name="clusterCreateRoleInstanceForm" property="role" value="${role}"/>
--%>
<tr>
Modified: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/clusters/ClusterAllocationHandler.java
===================================================================
--- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/clusters/ClusterAllocationHandler.java 2010-03-08 17:25:47 UTC (rev 8247)
+++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/clusters/ClusterAllocationHandler.java 2010-03-09 17:58:41 UTC (rev 8248)
@@ -115,8 +115,8 @@
}
NodeDeploymentService deploymentService = createNodeDeploymentService(instance);
String messages = deploymentService.deployApplication(name, descriptor.getComponentDescription(),
- controller.getRemoteLog());
- //null);
+ //controller.getRemoteLog());
+ null);
if (messages != null && !messages.isEmpty()) {
LOG.info(messages);
}
Modified: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/model/cluster/ClusterController.java
===================================================================
--- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/model/cluster/ClusterController.java 2010-03-08 17:25:47 UTC (rev 8247)
+++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/model/cluster/ClusterController.java 2010-03-09 17:58:41 UTC (rev 8248)
@@ -837,7 +837,9 @@
waitForFarmerAvailable();
try {
for (RoleAllocationRequest request : allocationRequests) {
- requestHosts(request, clusterAllocationCompleted);
+ if (!request.isEmpty()) {
+ requestHosts(request, clusterAllocationCompleted);
+ }
}
} catch (Throwable throwable) {
//failure, notify and rethrow
Modified: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/model/cluster/RoleAllocationRequest.java
===================================================================
--- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/model/cluster/RoleAllocationRequest.java 2010-03-08 17:25:47 UTC (rev 8247)
+++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/model/cluster/RoleAllocationRequest.java 2010-03-09 17:58:41 UTC (rev 8248)
@@ -62,6 +62,14 @@
return max;
}
+ /**
+ * Is the request empty?
+ * @return true if no hosts are expected
+ */
+ public boolean isEmpty() {
+ return min <= 0 && max == 0;
+ }
+
public boolean isStarted() {
return started;
}
Modified: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/servlets/HostsAndRolesServlet.java
===================================================================
--- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/servlets/HostsAndRolesServlet.java 2010-03-08 17:25:47 UTC (rev 8247)
+++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/web/servlets/HostsAndRolesServlet.java 2010-03-09 17:58:41 UTC (rev 8248)
@@ -82,9 +82,6 @@
//publish the sizes
properties.put(prefix + ROLE + "." + SIZE, "" + roleList.size());
-
-
-
//run through the role list
Map<String, HostInstanceList> roleMap = new HashMap<String, HostInstanceList>(roleList.size());
@@ -118,7 +115,7 @@
//now for each role, push out the hosts
for (String roleName: roleMap.keySet()) {
HostInstanceList hostList = roleMap.get(roleName);
- String roleprefix = prefix + "." + roleName + ".";
+ String roleprefix = prefix + ROLE + "." + roleName + ".";
properties.put(roleprefix + SIZE, ""+ hostList.size());
count = 1;
for (HostInstance host : hostList) {
Modified: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/common/ClusterFarmerUtils.java
===================================================================
--- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/common/ClusterFarmerUtils.java 2010-03-08 17:25:47 UTC (rev 8247)
+++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/common/ClusterFarmerUtils.java 2010-03-09 17:58:41 UTC (rev 8248)
@@ -42,6 +42,7 @@
* {@value}
*/
public static final String NEGATIVE_VALUES_NOT_SUPPORTED = "Negative values not supported";
+ private static final String ERROR_NO_FACTORY = "No Deployment Factory is defined for this farmer ";
/**
* check the min and max arguments
@@ -59,40 +60,72 @@
}
}
+ /**
+ * Get the diagnostics text. If the request for diagnostics fails, that exception's string value is returned
+ * instead
+ *
+ * @param farmer farmer to query
+ * @return some diagnostics
+ */
public static String getDiagnosticsTextRobustly(ClusterFarmer farmer) {
String text;
try {
text = farmer.getDiagnosticsText();
- } catch (IOException ignored) {
- text = "";
- } catch (SmartFrogException ignored) {
- text = "";
+ } catch (IOException e) {
+ text = e.toString();
+ } catch (SmartFrogException e) {
+ text = e.toString();
}
return text;
}
- public static NodeDeploymentService createNodeDeploymentService(ClusterFarmer farmer, ClusterNode node,
- NodeDeploymentServiceFactory factory)
+ /**
+ * Get the deployment service for a factory.
+ *
+ * @param farmer farmer
+ * @param node node to get a deployment service for
+ * @param factory the factory, can be null
+ * @return a deployment factory
+ * @throws IOException IO/network problems
+ * @throws SmartFrogException anything else, including the factory parameter being null
+ */
+ public static NodeDeploymentService createNodeDeploymentService(ClusterFarmer farmer,
+ ClusterNode node,
+ NodeDeploymentServiceFactory factory)
throws IOException, SmartFrogException {
if (factory == null) {
String text = getDiagnosticsTextRobustly(farmer);
- throw new SmartFrogDeploymentException("No Deployment Factory is defined for this farmer " + text);
+ throw new SmartFrogDeploymentException(ERROR_NO_FACTORY + text);
}
NodeDeploymentService instance = factory.createInstance(node);
return instance;
}
+ /**
+ * Check deployment is supported. Requires factory to not be null, and for it to support deployment
+ *
+ * @param factory service factory to work with
+ * @return true if deployment is allowed
+ * @throws IOException IO problems
+ * @throws SmartFrogException other problems
+ */
public static boolean isDeploymentServiceAvailable(NodeDeploymentServiceFactory factory)
throws IOException, SmartFrogException {
return factory != null && factory.isNodeDeploymentSupported();
}
-
-
-
+
+ /**
+ * Get the diagnostics for a factory
+ *
+ * @param factory service factory to work with
+ * @return the text
+ * @throws IOException IO problems
+ * @throws SmartFrogException other problems
+ */
public static String getNodeDeploymentServiceDiagnostics(NodeDeploymentServiceFactory factory)
throws SmartFrogException, IOException {
- if (factory==null) {
+ if (factory == null) {
return "No Deployment Service defined";
}
return factory.getDiagnosticsText();
Modified: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/deployment/NodeDeploymentOverSSH.java
===================================================================
--- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/deployment/NodeDeploymentOverSSH.java 2010-03-08 17:25:47 UTC (rev 8247)
+++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/deployment/NodeDeploymentOverSSH.java 2010-03-09 17:58:41 UTC (rev 8248)
@@ -79,7 +79,7 @@
*/
private static final int STARTUP_PING_SLEEP_TIME = 30;
private static final String ERROR_NO_EXECUTABLE = " Error: no executable ";
- private static final int SLEEP_TIME_FOR_HOSTNAME_RESOLUTION = 500;
+ private static final int SLEEP_TIME_FOR_HOSTNAME_RESOLUTION = 100;
public NodeDeploymentOverSSH(NodeDeploymentOverSSHFactory factory, ClusterNode node) {
super(node);
@@ -145,14 +145,18 @@
info(remoteLog, messages, "Deploying application with SSH to role '" + name + "' to " + connectionDetails);
//make a pre-emptive connection to the port; this blocks waiting for things like machines to come up
+
+ int connectTimeout = factory.getPortConnectTimeout();
+ info(remoteLog, messages, "Waiting "+ connectTimeout +"ms for the hostname " + hostname +" to resolve");
try {
- PortUtils.waitForHostnameToResolve(hostname, factory.getPortConnectTimeout(),
+ PortUtils.waitForHostnameToResolve(hostname, connectTimeout,
SLEEP_TIME_FOR_HOSTNAME_RESOLUTION);
} catch (InterruptedException e) {
throw (InterruptedIOException) new InterruptedIOException("Interrupted waiting for " + hostname)
.initCause(e);
}
- PortUtils.checkPort(hostname, factory.getPort(), factory.getPortConnectTimeout());
+ info(remoteLog, messages, "Waiting "+ connectTimeout +"ms for the SSH port "+ factory.getPort() + " to be open");
+ PortUtils.checkPort(hostname, factory.getPort(), connectTimeout);
Session session = null;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|