From: <st...@us...> - 2009-11-16 17:52:34
|
Revision: 8046 http://smartfrog.svn.sourceforge.net/smartfrog/?rev=8046&view=rev Author: steve_l Date: 2009-11-16 17:49:48 +0000 (Mon, 16 Nov 2009) Log Message: ----------- SFOS-1359 ssh deployment has worked now, improving... Modified Paths: -------------- trunk/core/components/cloudfarmer/build.xml trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/components/FarmCustomerImpl.java trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/deployment/NodeDeploymentOverSSHFactory.java trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/deployment/components.sf trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/ec2_hadoop_farmer.sf trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/manual_localhost.sf trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/mock/components.sf trunk/core/components/cloudfarmer/test/org/smartfrog/services/cloudfarmer/test/deployment/components.sf Added Paths: ----------- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/forkedhadoopcustomer.sf trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/hadoopcustomer.sf trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/localhost_over_ssh.sf Modified: trunk/core/components/cloudfarmer/build.xml =================================================================== --- trunk/core/components/cloudfarmer/build.xml 2009-11-16 16:21:37 UTC (rev 8045) +++ trunk/core/components/cloudfarmer/build.xml 2009-11-16 17:49:48 UTC (rev 8046) @@ -110,6 +110,21 @@ </deploy> </target> + <target name="localfarmer" depends="ready-to-deploy" + description="Deploy a farm manager that serves up the local host" > + <deploy> + <application name="farmer" + descriptor="/org/smartfrog/services/cloudfarmer/server/examples/manual_localhost.sf"/> + </deploy> + </target> + <target name="localfarmerssh" depends="ready-to-deploy" + description="Deploy a farm manager that serves up the local hostover SSH" > + <deploy> + <application name="farmer" + descriptor="/org/smartfrog/services/cloudfarmer/server/examples/localhost_over_ssh.sf"/> + </deploy> + </target> + <target name="mombasa" depends="ready-to-deploy" description="Deploy the mombasa web application" > <deploy> @@ -137,4 +152,13 @@ descriptor="/org/smartfrog/services/cloudfarmer/server/examples/ec2_hadoop_farmer.sf"/> </deploy> </target> + + <target name="forkedcluster" depends="ready-to-deploy" + description="Deploy a forked Hadoop cluster against a local farmer" > + <deploy> + <application name="forkedcluster" + descriptor="/org/smartfrog/services/cloudfarmer/server/examples/forkedhadoopcustomer.sf"/> + </deploy> + </target> + </project> \ No newline at end of file Modified: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/components/FarmCustomerImpl.java =================================================================== --- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/components/FarmCustomerImpl.java 2009-11-16 16:21:37 UTC (rev 8045) +++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/client/components/FarmCustomerImpl.java 2009-11-16 17:49:48 UTC (rev 8046) @@ -99,8 +99,8 @@ * Check the nodes are there on a liveness call * * @param source source of call - * @throws SmartFrogLivenessException - * @throws RemoteException + * @throws SmartFrogLivenessException failure to find a node in that role + * @throws RemoteException network problems */ @Override public void sfPing(Object source) throws SmartFrogLivenessException, RemoteException { @@ -140,6 +140,9 @@ nodes = null; } + /** + * This is the customer thread that pushed out the files + */ public class CustomerThread extends WorkflowThread { /** Modified: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/deployment/NodeDeploymentOverSSHFactory.java =================================================================== --- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/deployment/NodeDeploymentOverSSHFactory.java 2009-11-16 16:21:37 UTC (rev 8045) +++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/deployment/NodeDeploymentOverSSHFactory.java 2009-11-16 17:49:48 UTC (rev 8046) @@ -20,6 +20,7 @@ import java.io.IOException; import java.rmi.RemoteException; import java.util.ArrayList; +import java.util.Random; /** * This factory copies applicatiions over then starts them. @@ -28,16 +29,16 @@ implements NodeDeploymentServiceFactory, SSHComponent { private String destDir; - protected JSch jschInstance; + private JSch jschInstance; public static final String ATTR_DEST_DIR = "destDir"; public static final String ATTR_LOG_LEVEL = "logLevel"; public static final String ATTR_TEMP_FILE_PREFIX = "tempfilePrefix"; public static final String ATTR_KEEP_FILES = "keepFiles"; - protected int outputLogLevel; - protected int counter = 0; - protected String tempfilePrefix; + private int outputLogLevel; + private String tempfilePrefix; + private Random random = new Random(); - protected boolean keepFiles = false; + private boolean keepFiles = false; public NodeDeploymentOverSSHFactory() throws RemoteException { @@ -51,7 +52,7 @@ * @throws RemoteException network problems */ @Override - public void sfStart() throws SmartFrogException, RemoteException { + public synchronized void sfStart() throws SmartFrogException, RemoteException { super.sfStart(); destDir = sfResolve(ATTR_DEST_DIR, "", true); if (!destDir.endsWith("/")) { @@ -121,27 +122,35 @@ throws IOException, SmartFrogException { File localtempfile = File.createTempFile(tempfilePrefix, SF_SUFFIX); - String desttempfile = destDir + tempfilePrefix + incCounter() + SF_SUFFIX; + String desttempfile = destDir + tempfilePrefix + getNextNumber() + SF_SUFFIX; SFExpandFully.saveCDtoFile(cd, localtempfile); ArrayList<File> sourceFiles = new ArrayList<File>(); ArrayList<String> destFiles = new ArrayList<String>(); sourceFiles.add(localtempfile); destFiles.add(desttempfile); - String connectionDetails = hostname + ":" + getPort(); + String connectionDetails = getConnectionDetails(); Session session = null; try { log.info("Deploying application "+ name + " to " + connectionDetails); session = demandCreateSession(hostname); - sshExec(session, "mkdir -p " + destDir, false); + ArrayList<String> commandsList = new ArrayList<String>(1); + commandsList.add("mkdir -p " + destDir); + commandsList.add("exit"); + sshExec(session, commandsList, true); + ScpTo scp = new ScpTo(sfLog()); //copy up the temp files + log.info("copying " + localtempfile + " to " + desttempfile); scp.doCopy(session, destFiles, sourceFiles); String sshCommand = "sfStart " + "localhost" + " " + name + " " + desttempfile; log.info("executing: " + sshCommand); - sshExec(session, sshCommand, true); - if(!keepFiles) { - sshExec(session, "rm " + desttempfile, true); + commandsList = new ArrayList<String>(1); + commandsList.add(sshCommand); + if (!keepFiles) { + commandsList.add("rm " + desttempfile); } + commandsList.add("exit"); + sshExec(session, commandsList, true); } catch (JSchException e) { log.error("Failed to upload to " + connectionDetails + " : " + e, e); throw forward(e, connectionDetails); @@ -153,20 +162,34 @@ } } - private synchronized int incCounter() { - return counter++; + private String getConnectionDetails() { + return hostname + ":" + getPort(); } + /** + * Get a new number + * @return a new number for use in filenames + */ + private synchronized int getNextNumber() { + return random.nextInt(); + } + private void sshExec(Session session, String commandString, boolean checkResponse) throws JSchException, IOException, SmartFrogException { + ArrayList<String> commandsList = new ArrayList<String>(1); + commandsList.add(commandString); + sshExec(session, commandsList, checkResponse); + } + + private int sshExec(Session session, ArrayList<String> commandsList, boolean checkResponse) + throws JSchException, IOException, SmartFrogException { SshCommand command = new SshCommand(sfLog(), null); OutputStreamLog outputStream = new OutputStreamLog(log, outputLogLevel); - ArrayList<String> commandsList = new ArrayList<String>(1); - commandsList.add(commandString); int exitCode = command.execute(session, commandsList, outputStream, getTimeout()); - if (exitCode != 0) { - throw new SmartFrogException("Error response on command " + commandString); + if (checkResponse && exitCode != 0) { + throw new SmartFrogException("Error response on command " + commandsList.get(0)); } + return exitCode; } /** @@ -175,7 +198,7 @@ * @param session session, can be null */ private void endSession(Session session) { - if (session != null) { + if (session != null && session.isConnected()) { session.disconnect(); } } @@ -186,7 +209,23 @@ @Override public boolean terminateApplication(String name, boolean normal, String exitText) throws IOException, SmartFrogException { - return false; + Session session = null; + try { + log.info("Terminating application " + name + " for " + exitText); + session = demandCreateSession(hostname); + ArrayList<String> commandsList = new ArrayList<String>(1); + String sshCommand = "sfTerminate " + "localhost" + " " + name; + log.info("executing: " + sshCommand); + commandsList.add(sshCommand); + commandsList.add("exit"); + return sshExec(session, commandsList, false) == 0; + } catch (JSchException e) { + String connectionDetails = getConnectionDetails(); + log.error("Failed to terminate "+ name+ " on " + connectionDetails + " : " + e, e); + throw forward(e, connectionDetails); + } finally { + endSession(session); + } } /** Modified: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/deployment/components.sf =================================================================== --- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/deployment/components.sf 2009-11-16 16:21:37 UTC (rev 8045) +++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/deployment/components.sf 2009-11-16 17:49:48 UTC (rev 8046) @@ -49,7 +49,7 @@ destDir "/tmp"; //there for the schema host ""; - tempfilePrefix "sfoverssh"; + tempfilePrefix "sf_over_ssh_"; //log level for communications logLevel LogLevel:LOG_LEVEL_INFO; //passwordProvider TBD; Modified: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/ec2_hadoop_farmer.sf =================================================================== --- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/ec2_hadoop_farmer.sf 2009-11-16 16:21:37 UTC (rev 8045) +++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/ec2_hadoop_farmer.sf 2009-11-16 17:49:48 UTC (rev 8046) @@ -67,4 +67,7 @@ } + + + sfConfig extends EC2HadoopFarmer; \ No newline at end of file Added: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/forkedhadoopcustomer.sf =================================================================== --- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/forkedhadoopcustomer.sf (rev 0) +++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/forkedhadoopcustomer.sf 2009-11-16 17:49:48 UTC (rev 8046) @@ -0,0 +1,35 @@ +/* (C) Copyright 2009 Hewlett-Packard Development Company, LP + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +For more information: www.smartfrog.org + +*/ +#include "/org/smartfrog/services/cloudfarmer/server/examples/hadoopcustomer.sf" +#include "/org/smartfrog/services/cloudfarmer/server/manual/components.sf" +#include "/org/smartfrog/services/cloudfarmer/server/examples/hadooproles.sf" +#include "/org/smartfrog/extras/hadoop/cluster/components.sf" + +ForkedClusterFarmCustomer extends HadoopMasterFarmCustomer { + toDeployName "HadoopCluster"; + toDeploy extends HadoopClusterMultiProcess { + + } +} + + +sfConfig extends ForkedClusterFarmCustomer { + farmer LAZY PROCESS:farmer; +} \ No newline at end of file Added: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/hadoopcustomer.sf =================================================================== --- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/hadoopcustomer.sf (rev 0) +++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/hadoopcustomer.sf 2009-11-16 17:49:48 UTC (rev 8046) @@ -0,0 +1,43 @@ +/* (C) Copyright 2009 Hewlett-Packard Development Company, LP + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +For more information: www.smartfrog.org + +*/ +#include "/org/smartfrog/services/cloudfarmer/client/components/components.sf" + +HadoopMasterFarmCustomer extends WorkflowFarmCustomer { + role "master"; + toDeployName "HadoopMaster"; + toDeploy extends Compound { + + } + min 1; + expected 1; + deleteOnTerminate false; +} + +HadoopWorkerFarmCustomer extends WorkflowFarmCustomer { + role "worker"; + toDeployName "HadoopWorker"; + toDeploy extends Compound { + + } + min 1; + expected 1; + deleteOnTerminate false; +} + Copied: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/localhost_over_ssh.sf (from rev 8042, trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/manual_localhost.sf) =================================================================== --- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/localhost_over_ssh.sf (rev 0) +++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/localhost_over_ssh.sf 2009-11-16 17:49:48 UTC (rev 8046) @@ -0,0 +1,22 @@ +/* (C) Copyright 2009 Hewlett-Packard Development Company, LP + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +For more information: www.smartfrog.org + +*/ +#include "/org/smartfrog/services/cloudfarmer/server/examples/manual_localhost.sf" + +sfConfig extends LocalhostClusterSSHDeploy; \ No newline at end of file Modified: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/manual_localhost.sf =================================================================== --- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/manual_localhost.sf 2009-11-16 16:21:37 UTC (rev 8045) +++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/examples/manual_localhost.sf 2009-11-16 17:49:48 UTC (rev 8046) @@ -54,11 +54,6 @@ LocalhostClusterSSHDeploy extends LocalhostCluster { description "a localhost cluster with SmartFrog-RMI deployment" - passwordProvider extends FilePassword { - passwordFile ((LAZY PROPERTY user.home) ++ "/.ssh/id_dsa"); - } - - deploymentFactory extends NodeDeploymentOverSSHFactory { username LAZY PROPERTY user.name; authentication authenticate_key; @@ -67,4 +62,4 @@ } -sfConfig extends LocalhostCluster; \ No newline at end of file +sfConfig extends LocalhostClusterRMIDeploy; \ No newline at end of file Modified: trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/mock/components.sf =================================================================== --- trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/mock/components.sf 2009-11-16 16:21:37 UTC (rev 8045) +++ trunk/core/components/cloudfarmer/src/org/smartfrog/services/cloudfarmer/server/mock/components.sf 2009-11-16 17:49:48 UTC (rev 8046) @@ -50,7 +50,7 @@ externalDomain "external"; //delay per node - nodeStartupDelayMilliseconds 1000; + nodeStartupDelayMilliseconds 100; //is the cluster actually available available true; Modified: trunk/core/components/cloudfarmer/test/org/smartfrog/services/cloudfarmer/test/deployment/components.sf =================================================================== --- trunk/core/components/cloudfarmer/test/org/smartfrog/services/cloudfarmer/test/deployment/components.sf 2009-11-16 16:21:37 UTC (rev 8045) +++ trunk/core/components/cloudfarmer/test/org/smartfrog/services/cloudfarmer/test/deployment/components.sf 2009-11-16 17:49:48 UTC (rev 8046) @@ -19,18 +19,15 @@ */ #include "/org/smartfrog/services/cloudfarmer/test/components.sf" #include "/org/smartfrog/services/cloudfarmer/server/examples/manual_localhost.sf" +#include "/org/smartfrog/services/cloudfarmer/server/examples/hadoopcustomer.sf" -DeployMaster extends WorkflowFarmCustomer { - role "master"; - toDeploy extends DATA { - sfClass Compound:sfClass; +DeployMaster extends HadoopMasterFarmCustomer { + toDeploy extends Compound { description "this is the toplevel application"; + deployed true; } - min 1; - expected 1; - deleteOnTerminate false; } @@ -39,6 +36,9 @@ } TestLocalhostClusterSSH extends LocalhostClusterSSHDeploy { + + deploymentFactory:keepFiles true; + } testLocalhostDeployCompoundOverRMI extends ExpectDeploy { @@ -55,6 +55,12 @@ farmer LAZY action; toDeployName "overRMI"; } + + assert extends AssertWorkflow { + reference LAZY ROOT:overRMI; + attribute "deployed"; + attributeEquals "true"; + } } } This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |