Menu
▾
▴
[Smartfrog-checkins] SF.net SVN: smartfrog:[7436] trunk/core/components/hadoop
|
From: <st...@us...> - 2009-03-03 17:46:43
|
Revision: 7436
http://smartfrog.svn.sourceforge.net/smartfrog/?rev=7436&view=rev
Author: steve_l
Date: 2009-03-03 17:46:39 +0000 (Tue, 03 Mar 2009)
Log Message:
-----------
SFOS-1135 move hadoop-cluster components to using dynamically determined hostnames
Modified Paths:
--------------
trunk/core/components/hadoop/build.xml
trunk/core/components/hadoop/doc/managing_hadoop.ppt
trunk/core/components/hadoop/src/org/apache/hadoop/hdfs/server/namenode/ExtNameNode.java
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/common/DfsUtils.java
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/HadoopServiceImpl.java
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/cluster-driven.sf
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/lightweight.sf
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/options.sf
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/testjob.sf
trunk/core/components/hadoop/test/org/smartfrog/services/hadoop/test/system/local/components.sf
Modified: trunk/core/components/hadoop/build.xml
===================================================================
--- trunk/core/components/hadoop/build.xml 2009-03-03 17:34:45 UTC (rev 7435)
+++ trunk/core/components/hadoop/build.xml 2009-03-03 17:46:39 UTC (rev 7436)
@@ -36,7 +36,10 @@
<!--we are a component -->
<property name="is.component" value="true"/>
<property name="ivy.enabled" value="true"/>
- <property name="junit.timeout" value="120000000"/>
+ <property name="junit.timeout" value="12000000"/>
+ <!--time for a single test to execute-->
+ <property name="test.timeout.execute" value="60000"/>
+
<!-- override point -->
<property file="build.properties"/>
Modified: trunk/core/components/hadoop/doc/managing_hadoop.ppt
===================================================================
(Binary files differ)
Modified: trunk/core/components/hadoop/src/org/apache/hadoop/hdfs/server/namenode/ExtNameNode.java
===================================================================
--- trunk/core/components/hadoop/src/org/apache/hadoop/hdfs/server/namenode/ExtNameNode.java 2009-03-03 17:34:45 UTC (rev 7435)
+++ trunk/core/components/hadoop/src/org/apache/hadoop/hdfs/server/namenode/ExtNameNode.java 2009-03-03 17:46:39 UTC (rev 7436)
@@ -168,6 +168,9 @@
protected void onStateChange(ServiceState oldState, ServiceState newState) {
super.onStateChange(oldState, newState);
LOG.info("State change: NameNode is now "+ newState);
+ //when we go live, we also push out our new URL
+
+ //tell anyone listening
notifier.onStateChange(oldState, newState);
}
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/common/DfsUtils.java
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/common/DfsUtils.java 2009-03-03 17:34:45 UTC (rev 7435)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/common/DfsUtils.java 2009-03-03 17:46:39 UTC (rev 7436)
@@ -45,7 +45,7 @@
public class DfsUtils {
public static final String ERROR_INVALID_FILESYSTEM_URI = "Invalid " + HadoopConfiguration.FS_DEFAULT_NAME
+ " URI: ";
- public static final String ERROR_FAILED_TO_INITIALISE_FILESYSTEM = "Failed to initialise filesystem";
+ public static final String ERROR_FAILED_TO_INITIALISE_FILESYSTEM = "Failed to initialise filesystem: ";
public static final String ERROR_FAILED_TO_DELETE_PATH = "Failed to delete path ";
public static final String ERROR_FAILED_TO_CLOSE = "Failed to close ";
public static final String ERROR_MKDIR_FAILED = "Unable to create the destination directories for ";
@@ -121,7 +121,7 @@
*/
public static FileSystem createFileSystem(String filesystemURL, ManagedConfiguration conf)
throws SFHadoopException {
- URI uri = null;
+ URI uri;
try {
uri = new URI(filesystemURL);
} catch (URISyntaxException e) {
@@ -137,7 +137,7 @@
return dfs;
} catch (IOException e) {
SFHadoopException hadoopException = SFHadoopException
- .forward(ERROR_FAILED_TO_INITIALISE_FILESYSTEM, e);
+ .forward(ERROR_FAILED_TO_INITIALISE_FILESYSTEM + filesystemURL, e);
hadoopException.addConfiguration(conf);
throw hadoopException;
}
@@ -246,7 +246,9 @@
throw new SmartFrogRuntimeException(ERROR_MKDIR_FAILED + dest);
}
} catch (IOException e) {
- throw new SmartFrogRuntimeException(ERROR_MKDIR_FAILED + dest + " : " + e, e);
+ throw new SmartFrogRuntimeException(ERROR_MKDIR_FAILED + dest
+ + " in " + fileSystem.getUri()
+ + " : " + e, e);
}
}
@@ -270,15 +272,16 @@
int blocksize) throws SmartFrogRuntimeException {
assertNotDependent(srcFS, src, dstFS, dst);
FileStatus status;
+ URI fsuri = srcFS.getUri();
try {
status = srcFS.getFileStatus(src);
} catch (FileNotFoundException fe) {
- throw new SmartFrogRuntimeException(ERROR_MISSING_SOURCE_FILE + src + " in " + srcFS.getUri(), fe);
+ throw new SmartFrogRuntimeException(ERROR_MISSING_SOURCE_FILE + src + " in " + fsuri, fe);
} catch (IOException e) {
- throw new SmartFrogRuntimeException(ERROR_NO_STAT + src + " in " + srcFS.getUri() + " : " + e, e);
+ throw new SmartFrogRuntimeException(ERROR_NO_STAT + src + " in " + fsuri + " : " + e, e);
}
if (status.isDir()) {
- throw new SmartFrogRuntimeException(ERROR_NO_DIRECTORY_COPY + src + " in " + srcFS.getUri());
+ throw new SmartFrogRuntimeException(ERROR_NO_DIRECTORY_COPY + src + " in " + fsuri);
}
InputStream in = null;
OutputStream out = null;
@@ -293,7 +296,7 @@
try {
IOUtils.copyBytes(in, out, blocksize, true);
} catch (IOException e) {
- throw new SmartFrogRuntimeException(ERROR_COPY_FAILED + src + " in " + srcFS.getUri()
+ throw new SmartFrogRuntimeException(ERROR_COPY_FAILED + src + " in " + fsuri
+ " to " + dst + " in " + dstFS.getUri()
+ " : " + e,
e);
@@ -320,7 +323,9 @@
fileSystem.copyFromLocalFile(false, overwrite, localSource, dest);
} catch (IOException e) {
throw new SmartFrogRuntimeException(
- FAILED_TO_COPY + source + " to " + dest, e);
+ FAILED_TO_COPY + source + " to " + dest
+ + " on " + fileSystem.getUri(),
+ e);
}
}
@@ -351,7 +356,7 @@
srcFstatus = fileSystem.getFileStatus(src);
} catch (FileNotFoundException e) {
FileNotFoundException fnf = new FileNotFoundException(src +
- ": No such file or directory");
+ ": No such file or directory in " + fileSystem.getUri());
fnf.initCause(e);
throw fnf;
}
@@ -363,11 +368,13 @@
if ((srcFstatus != null) && (dstFstatus != null)) {
if (srcFstatus.isDir() && !dstFstatus.isDir()) {
throw new IOException("cannot overwrite non directory "
- + dstPath + " with directory " + srcPath);
+ + dstPath + " with directory " + srcPath
+ + " in " + fileSystem.getUri());
}
}
throw new IOException("Failed to rename '" + srcPath
- + "' to '" + dstPath + "'");
+ + "' to '" + dstPath + "'"
+ + " in " + fileSystem.getUri());
}
}
}
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/HadoopServiceImpl.java
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/HadoopServiceImpl.java 2009-03-03 17:34:45 UTC (rev 7435)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/HadoopServiceImpl.java 2009-03-03 17:46:39 UTC (rev 7436)
@@ -68,6 +68,7 @@
private List<PortEntry> portList;
private static final int CONNECT_TIMEOUT = 2000;
private boolean serviceStartupInProgress;
+ protected static final String LIVE_ATTRIBUTE_PREFIX = "live.";
protected HadoopServiceImpl() throws RemoteException {
@@ -541,9 +542,7 @@
if (s != null && !sfIsTerminated) {
sfLog().info(getServiceName() + " deployment complete: service is: " + s);
//now we copy over the port values
- ServiceInfo si = getServiceInfo();
- List<BindingTuple> bindingTupleList = si.getBindingInformation();
- copyBindingList(this, "live." ,bindingTupleList);
+ publishBindingInfo(LIVE_ATTRIBUTE_PREFIX);
} else {
String message = getServiceName() + " deployment completed after component was terminated."
+ "Hadoop service is " + hadoopService;
@@ -556,6 +555,18 @@
}
/**
+ * Publish the service's binding information to this component, with the specified prefix
+ * @param prefix prefix -can be null
+ * @throws IOException IO/hadoop problems
+ * @throws SmartFrogException smartfrog problems
+ */
+ protected void publishBindingInfo(String prefix) throws SmartFrogRuntimeException, RemoteException {
+ ServiceInfo si = getServiceInfo();
+ List<BindingTuple> bindingTupleList = si.getBindingInformation();
+ copyBindingList(this, prefix, bindingTupleList);
+ }
+
+ /**
* Copy the binding list
* @param target prim to update
* @param prefix prefix to insert for every attribute (use "" for none)
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/cluster-driven.sf
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/cluster-driven.sf 2009-03-03 17:34:45 UTC (rev 7435)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/cluster-driven.sf 2009-03-03 17:46:39 UTC (rev 7436)
@@ -36,7 +36,7 @@
dfs.datanode.handler.count ClientHandlerThreads;
nameNodeURL NAME_NODE_URL;
dataNodeURL DATA_NODE_URL ;
- hostaddress HOSTNAME;
+ hostaddress LOCALHOST;
any_port (hostaddress ++ ":0");
dfs.datanode.address any_port;
dfs.datanode.http.address any_port;
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/lightweight.sf
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/lightweight.sf 2009-03-03 17:34:45 UTC (rev 7435)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/lightweight.sf 2009-03-03 17:46:39 UTC (rev 7436)
@@ -52,11 +52,10 @@
smartfrog.dump.conf DUMP_CONF;
dfs.heartbeat.interval 1;
dfs.datanode.handler.count ClientHandlerThreads;
- hostaddress HOSTNAME;
- any_port (hostaddress ++ ":0");
- dfs.datanode.http.address (hostaddress ++ ":" ++ DataNodeHttpPort);
- dfs.datanode.https.address (hostaddress ++ ":" ++ DataNodeHttpsPort);
- dfs.datanode.address (hostaddress ++ ":" ++ DataNodeIpcPort);
+ hostname LOCALHOST;
+ dfs.datanode.http.address (hostname ++ ":" ++ DataNodeHttpPort);
+ dfs.datanode.https.address (hostname ++ ":" ++ DataNodeHttpsPort);
+ dfs.datanode.address (hostname ++ ":" ++ DataNodeIpcPort);
dfs.blockreport.intervalMsec 10000;
dfs.blockreport.initialDelay 0;
@@ -70,8 +69,10 @@
HadoopCompoundBase extends Compound {
fs.default.name FS.DEFAULT.NAME;
+/*
nameNodeURL NAME_NODE_URL;
dataNodeURL DATA_NODE_IPC_URL ;
+*/
mapred.job.tracker HadoopConfiguration:mapred.job.tracker;
logDir extends TempDir {
@@ -101,6 +102,13 @@
cluster LAZY namenode;
+ hostname LOCALHOST;
+ IpcPort NameNodeIpcPort;
+ HttpPort NameNodeHttpPort
+
+ fs.default.name ("hdfs://" ++ hostname ++ ":" ++ NameNodeIpcPort );
+ namenode.URL ("http://" ++ hostname ++ ":" ++ HttpPort );
+
namenode extends LightweightNameNode {
fs.default.name PARENT:fs.default.name;
dataDir LAZY PARENT:dataDir;
@@ -110,9 +118,10 @@
nameDirectories [nameDirectory];
logDir LAZY PARENT:logDir;
dfs.namenode.startup ACTION_FORMAT;
- dfs.http.address nameNodeURL;
+ dfs.http.address namenode.URL;
dfs.permissions false;
minWorkerCount 0;
+
}
}
@@ -122,15 +131,17 @@
TransientDataNodeCompound extends HadoopCompoundBase {
description "A data node bound to a temporary directory";
+ hostname LOCALHOST;
+
localDataDir extends TempDir {
}
datanode extends LightweightDataNode {
+ hostname PARENT:hostname;
fs.default.name PARENT:fs.default.name;
dataDirectories [LAZY localDataDir];
logDir LAZY PARENT:logDir;
- dfs.datanode.address dataNodeURL ;
}
}
@@ -147,10 +158,11 @@
}
datanode extends LightweightDataNode {
+ hostname PARENT:hostname;
fs.default.name PARENT:fs.default.name;
dataDirectories [LAZY localDataDir];
logDir LAZY PARENT:logDir;
- dfs.datanode.address dataNodeURL ;
+
}
}
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/options.sf
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/options.sf 2009-03-03 17:34:45 UTC (rev 7435)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/options.sf 2009-03-03 17:46:39 UTC (rev 7436)
@@ -32,23 +32,23 @@
NameNodeIpcPort "8020";
NameNodeHttpPort "8021";
-DataNodeHttpPort "8022";
-DataNodeHttpsPort "8023";
-DataNodeIpcPort "8024";
+DataNodeHttpPort "8030";
+DataNodeHttpsPort "8031";
+DataNodeIpcPort "8042";
-DataNodeHttpPort2 "8032";
-DataNodeHttpsPort2 "8033";
-DataNodeIpcPort2 "8034";
+DataNodeHttpPort2 "8040";
+DataNodeHttpsPort2 "8041";
+DataNodeIpcPort2 "8042";
HandlerThreads 4;
ClientHandlerThreads 2;
-HOSTNAME "localhost";
-FS.DEFAULT.NAME ("hdfs://" ++ HOSTNAME ++ ":" ++ NameNodeIpcPort );
-NAME_NODE_URL ("http://" ++ HOSTNAME ++ ":" ++ NameNodeHttpPort );
-DATA_NODE_URL ("http://" ++ HOSTNAME ++ ":" ++ DataNodeHttpPort );
-DATA_NODE_IPC_URL ("http://" ++ HOSTNAME ++ ":" ++ DataNodeIpcPort );
+LOCALHOST "localhost";
+FS.DEFAULT.NAME ("hdfs://" ++ LOCALHOST ++ ":" ++ NameNodeIpcPort );
+NAME_NODE_URL ("http://" ++ LOCALHOST ++ ":" ++ NameNodeHttpPort );
+DATA_NODE_URL ("http://" ++ LOCALHOST ++ ":" ++ DataNodeHttpPort );
+DATA_NODE_IPC_URL ("http://" ++ LOCALHOST ++ ":" ++ DataNodeIpcPort );
STARTUP_SLEEP_TIME 5000;
IPC_CONNECT_TIMEOUT 10000;
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/testjob.sf
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/testjob.sf 2009-03-03 17:34:45 UTC (rev 7435)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/testjob.sf 2009-03-03 17:46:39 UTC (rev 7436)
@@ -257,7 +257,9 @@
mapred.output.dir (fs.default.name ++ testDirOut);
mapred.working.dir (fs.default.name ++ testDirWorking);
- inputFile extends SmallCountingFile;
+ inputFile extends SmallCountingFile {
+ filename inputFileLocal;
+ }
CopyDataIn extends DfsCopyFileInWorkflow {
cluster LAZY PARENT:cluster;
Modified: trunk/core/components/hadoop/test/org/smartfrog/services/hadoop/test/system/local/components.sf
===================================================================
--- trunk/core/components/hadoop/test/org/smartfrog/services/hadoop/test/system/local/components.sf 2009-03-03 17:34:45 UTC (rev 7435)
+++ trunk/core/components/hadoop/test/org/smartfrog/services/hadoop/test/system/local/components.sf 2009-03-03 17:46:39 UTC (rev 7436)
@@ -30,6 +30,8 @@
OrphanDatanodeCompound extends LocalHadoopCompound {
+ hostname LOCALHOST;
+
description "A cluster with a task tracker but not job tracker";
localDataDir extends TempDir {
@@ -37,6 +39,7 @@
}
datanode extends LightweightDataNode {
+ hostname PARENT:hostname;
fs.default.name PARENT:fs.default.name;
dataDirectories [LAZY localDataDir];
logDir LAZY PARENT:logDir;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|