Menu
▾
▴
[Smartfrog-checkins] SF.net SVN: smartfrog: [6491] trunk/core/components/hadoop
|
From: <st...@us...> - 2008-05-07 22:33:36
|
Revision: 6491
http://smartfrog.svn.sourceforge.net/smartfrog/?rev=6491&view=rev
Author: steve_l
Date: 2008-05-07 15:33:34 -0700 (Wed, 07 May 2008)
Log Message:
-----------
SFOS-838 write component to start namenode in-VM
-tests and killing errors one-by-one
Modified Paths:
--------------
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/HadoopConfiguration.java
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/FileSystemNode.java
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/FileSystemNodeImpl.java
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/filesystemnode.sf
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/datanode/DatanodeImpl.java
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/datanode/datanode.sf
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/hadoopconfiguration.sf
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/namenode/NamenodeImpl.java
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/namenode/namenode.sf
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/conf/ConfigurationAttributes.java
trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/hplb.sf
trunk/core/components/hadoop/test/org/smartfrog/services/hadoop/test/system/local/components.sf
trunk/core/components/hadoop/test/org/smartfrog/services/hadoop/test/system/local/namenode/components.sf
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/HadoopConfiguration.java
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/HadoopConfiguration.java 2008-05-07 22:20:49 UTC (rev 6490)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/HadoopConfiguration.java 2008-05-07 22:33:34 UTC (rev 6491)
@@ -30,4 +30,5 @@
public interface HadoopConfiguration extends Remote, ConfigurationAttributes {
+
}
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/FileSystemNode.java
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/FileSystemNode.java 2008-05-07 22:20:49 UTC (rev 6490)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/FileSystemNode.java 2008-05-07 22:33:34 UTC (rev 6491)
@@ -20,12 +20,18 @@
package org.smartfrog.services.hadoop.components.cluster;
-import java.rmi.Remote;
+import org.smartfrog.services.hadoop.components.HadoopConfiguration;
+
/**
*
*/
-public interface FileSystemNode extends Remote {
+public interface FileSystemNode extends HadoopConfiguration {
String ATTR_MIN_DISK_SPACE="minDiskSpaceMB";
+ String ATTR_DATA_DIRECTORIES = "dataDirectories";
+ String ATTR_NAME_DIRECTORIES = "nameDirectories";
+ /** {@value} */
+ String ATTR_CHECK_RUNNING = "checkRunning";
+ String ATTR_LOG_DIR = "logDir";
}
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/FileSystemNodeImpl.java
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/FileSystemNodeImpl.java 2008-05-07 22:20:49 UTC (rev 6490)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/FileSystemNodeImpl.java 2008-05-07 22:33:34 UTC (rev 6491)
@@ -22,14 +22,104 @@
package org.smartfrog.services.hadoop.components.cluster;
import org.smartfrog.sfcore.prim.PrimImpl;
+import org.smartfrog.sfcore.common.SmartFrogException;
+import org.smartfrog.sfcore.common.SmartFrogDeploymentException;
+import org.smartfrog.sfcore.common.SmartFrogLifecycleException;
+import org.smartfrog.sfcore.reference.Reference;
+import org.smartfrog.services.hadoop.conf.ManagedConfiguration;
+import org.smartfrog.services.hadoop.components.HadoopConfiguration;
+import org.smartfrog.services.filesystem.FileSystem;
import java.rmi.RemoteException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Vector;
+import java.io.File;
/**
*
*/
public class FileSystemNodeImpl extends PrimImpl implements FileSystemNode {
+ private ManagedConfiguration configuration;
+ protected static final Reference DATA_DIRECTORIES = new Reference(
+ ATTR_DATA_DIRECTORIES);
+ protected static final Reference NAME_DIRECTORIES = new Reference(
+ ATTR_NAME_DIRECTORIES);
+
public FileSystemNodeImpl() throws RemoteException {
}
+
+ /**
+ * Can be called to start components. Subclasses should override to provide
+ * functionality Do not block in this call, but spawn off any main loops!
+ *
+ * @throws SmartFrogException failure while starting
+ * @throws RemoteException In case of network/rmi error
+ */
+ public synchronized void sfStart()
+ throws SmartFrogException, RemoteException {
+ super.sfStart();
+ //get the filesystem name, validate it
+ String filesystemName = sfResolve(FS_DEFAULT_NAME, "", true);
+ try {
+ URI uri=new URI(filesystemName);
+ if(uri.getPort()==-1) {
+ throw new SmartFrogLifecycleException("Undefined port on " + FS_DEFAULT_NAME + " value :" + filesystemName);
+ }
+ } catch (URISyntaxException e) {
+ throw new SmartFrogLifecycleException("Bad "+ FS_DEFAULT_NAME+ " value :"+filesystemName,e);
+ }
+
+
+ configuration = new ManagedConfiguration(this);
+ if(sfLog().isDebugEnabled()) {
+ sfLog().debug(configuration.dumpQuietly());
+ }
+ }
+
+ public ManagedConfiguration createConfiguration() {
+ return new ManagedConfiguration(this);
+ }
+
+ /**
+ * Run through the directories, create all that are there
+ * @param dirs list of directories
+ * @param createDirs create the directories?
+ * @return the directories all converted to a list split by commas
+ */
+ protected String createDirectoryList(Vector<String> dirs,boolean createDirs) {
+ StringBuilder path = new StringBuilder();
+ for (String dir : dirs) {
+ File directory = new File(dir);
+ if (createDirs) {
+ directory.mkdirs();
+ }
+ if (path.length() > 0) {
+ path.append(',');
+ }
+ path.append(directory.getAbsolutePath());
+ }
+ String value = path.toString();
+ return value;
+ }
+
+ /**
+ * Go from a list of paths/fileIntfs to a comma separated list, create
+ * directories on demand
+ * @param sourceRef source reference
+ * @param replaceAttribute attribute to replace
+ * @return the directories
+ * @throws SmartFrogException failure while starting
+ * @throws RemoteException In case of network/rmi error
+ */
+ protected Vector<String> createDirectoryListAttribute(Reference sourceRef,
+ String replaceAttribute)
+ throws RemoteException, SmartFrogException {
+ Vector<String> dirs;
+ dirs= FileSystem.resolveFileList(this, sourceRef,null,true,null);
+ String value = createDirectoryList(dirs,true);
+ sfReplaceAttribute(replaceAttribute, value);
+ return dirs;
+ }
}
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/filesystemnode.sf
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/filesystemnode.sf 2008-05-07 22:20:49 UTC (rev 6490)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/cluster/filesystemnode.sf 2008-05-07 22:33:34 UTC (rev 6491)
@@ -25,11 +25,11 @@
FileSystemNode extends ClusterNode {
//the set of operations of a node on startup
- FORMAT "FORMAT";
- REGULAR "REGULAR";
- UPGRADE "UPGRADE";
- ROLLBACK "ROLLBACK";
- FINALIZE "FINALIZE";
+ ACTION_FORMAT "FORMAT";
+ ACTION_REGULAR "REGULAR";
+ ACTION_UPGRADE "UPGRADE";
+ ACTION_ROLLBACK "ROLLBACK";
+ ACTION_FINALIZE "FINALIZE";
dfs.block.size (64 * 1024 * 1024);
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/datanode/DatanodeImpl.java
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/datanode/DatanodeImpl.java 2008-05-07 22:20:49 UTC (rev 6490)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/datanode/DatanodeImpl.java 2008-05-07 22:33:34 UTC (rev 6491)
@@ -43,8 +43,6 @@
public class DatanodeImpl extends FileSystemNodeImpl implements HadoopCluster {
private DataNode datanode;
- public static final String DATA_DIRECTORIES = "dataDirectories";
- private static final String DFS_DATA_DIR = "dfs.data.dir";
public DatanodeImpl() throws RemoteException {
}
@@ -57,25 +55,16 @@
*/
public synchronized void sfStart() throws SmartFrogException, RemoteException {
super.sfStart();
- Vector<String> dataDirs;
- ManagedConfiguration conf = new ManagedConfiguration(this);
- dataDirs = ListUtils.resolveStringList(this, new Reference(
- DATA_DIRECTORIES), true);
- StringBuilder path = new StringBuilder();
- for (String dir : dataDirs) {
- File directory = new File(dir);
- directory.mkdirs();
- if (path.length() > 0) {
- path.append(',');
- }
- path.append(directory.getAbsolutePath());
- }
- conf.set(DFS_DATA_DIR, path.toString());
+ createDirectoryListAttribute(DATA_DIRECTORIES, DFS_DATA_DIR);
+ ManagedConfiguration conf = createConfiguration();
try {
datanode = DataNode.run(conf);
} catch (IOException e) {
throw new SmartFrogException("Failed to start datanode: "
+ e.getMessage() + '\n' + conf.dumpQuietly(), e);
+ } catch (IllegalArgumentException e) {
+ throw new SmartFrogException("Failed to start datanode: "
+ + e.getMessage() + "\n" + conf.dumpQuietly(), e);
}
}
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/datanode/datanode.sf
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/datanode/datanode.sf 2008-05-07 22:20:49 UTC (rev 6490)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/datanode/datanode.sf 2008-05-07 22:33:34 UTC (rev 6491)
@@ -24,6 +24,7 @@
DataNode extends FileSystemNode {
+ sfClass "org.smartfrog.services.hadoop.components.datanode.DatanodeImpl";
dfs.balance.bandwidthPerSec (1024 * 1024);
dfs.blockreport.intervalMsec (60 * 60 * 1000);
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/hadoopconfiguration.sf
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/hadoopconfiguration.sf 2008-05-07 22:20:49 UTC (rev 6490)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/hadoopconfiguration.sf 2008-05-07 22:33:34 UTC (rev 6491)
@@ -41,12 +41,11 @@
HadoopConfiguration extends Prim {
- dfs.info.bindAddress NULL;
- dfs.info.port NULL;
- dfs.http.address NULL;
+ //dfs.http.address
+
//default filesystem name
- fs.default.name "file:///";
+// fs.default.name "file:///";
fs.local.block.size ((32 * 1024)* 1024);
/*
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/namenode/NamenodeImpl.java
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/namenode/NamenodeImpl.java 2008-05-07 22:20:49 UTC (rev 6490)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/namenode/NamenodeImpl.java 2008-05-07 22:33:34 UTC (rev 6491)
@@ -20,11 +20,14 @@
package org.smartfrog.services.hadoop.components.namenode;
import org.smartfrog.services.hadoop.components.HadoopCluster;
+import org.smartfrog.services.hadoop.components.HadoopConfiguration;
import org.smartfrog.services.hadoop.components.cluster.FileSystemNodeImpl;
+import org.smartfrog.services.hadoop.components.cluster.FileSystemNode;
import org.smartfrog.services.hadoop.conf.ManagedConfiguration;
import org.smartfrog.services.filesystem.FileSystem;
import org.smartfrog.sfcore.common.SmartFrogException;
import org.smartfrog.sfcore.common.SmartFrogLivenessException;
+import org.smartfrog.sfcore.common.SmartFrogRuntimeException;
import org.smartfrog.sfcore.prim.Liveness;
import org.smartfrog.sfcore.prim.PrimImpl;
import org.smartfrog.sfcore.prim.TerminationRecord;
@@ -42,14 +45,10 @@
* Created 06-May-2008 16:31:49
*/
-public class NamenodeImpl extends FileSystemNodeImpl implements HadoopCluster {
+public class NamenodeImpl extends FileSystemNodeImpl implements
+ FileSystemNode {
+
private ExtNameNode namenode;
- public static final String ATTR_DATA_DIRECTORIES = "dataDirectories";
- private static final String DFS_DATA_DIR = "dfs.data.dir";
- /** {@value} */
- public static final String ATTR_CHECK_RUNNING = "checkRunning";
- private static final Reference DATA_DIRECTORIES = new Reference(
- ATTR_DATA_DIRECTORIES);
public NamenodeImpl() throws RemoteException {
}
@@ -58,29 +57,27 @@
* Create the datanode
*
* @throws SmartFrogException failure while starting
- * @throws RemoteException In case of network/rmi error
+ * @throws RemoteException In case of network/rmi error
*/
- public synchronized void sfStart() throws SmartFrogException, RemoteException {
+ public synchronized void sfStart()
+ throws SmartFrogException, RemoteException {
super.sfStart();
- Vector<String> dataDirs;
- ManagedConfiguration conf = new ManagedConfiguration(this);
- dataDirs= FileSystem.resolveFileList(this,DATA_DIRECTORIES,null,true,null);
- StringBuilder path = new StringBuilder();
- for (String dir : dataDirs) {
- File directory = new File(dir);
- directory.mkdirs();
- if (path.length() > 0) {
- path.append(',');
- }
- path.append(directory.getAbsolutePath());
- }
- conf.set(DFS_DATA_DIR, path.toString());
+ createDirectoryListAttribute(DATA_DIRECTORIES, DFS_DATA_DIR);
+ createDirectoryListAttribute(NAME_DIRECTORIES, DFS_NAME_DIR);
+ File logDir = FileSystem.lookupAbsoluteFile(this,
+ ATTR_LOG_DIR, null, null, true, null);
+ logDir.mkdirs();
+ sfReplaceAttribute(HADOOP_LOG_DIR, logDir.getAbsolutePath());
+ ManagedConfiguration conf = createConfiguration();
try {
namenode = ExtNameNode.createNameNode(conf);
} catch (IOException e) {
throw new SmartFrogException("Failed to start namenode: "
+ e.getMessage() + "\n" + conf.dumpQuietly(), e);
+ } catch (IllegalArgumentException e) {
+ throw new SmartFrogException("Failed to start namenode: "
+ + e.getMessage() + "\n" + conf.dumpQuietly(), e);
}
}
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/namenode/namenode.sf
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/namenode/namenode.sf 2008-05-07 22:20:49 UTC (rev 6490)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/components/namenode/namenode.sf 2008-05-07 22:33:34 UTC (rev 6491)
@@ -36,9 +36,25 @@
NameNode extends FileSystemNode {
+ sfClass "org.smartfrog.services.hadoop.components.namenode.NamenodeImpl";
+
+ //name directory: a List of directories
+ nameDirectories TBD;
+ //the data directories, again a list of directories
+ dataDirectories TBD;
+
+ //log directory
+ logDir TBD;
+
//startup mode
- dfs.namenode.sartup REGULAR;
+ dfs.namenode.startup ACTION_REGULAR;
+ dfs.http.address "http://localhost:8021";
+
+ //this is set from logDir
+ //hadoop.log.dir TBD
+
+
//how many handler threads
dfs.namenode.handler.count 10;
//fs.default.name
@@ -71,10 +87,9 @@
/** Safe mode extension after the threshold. */
dfs.safemode.extension 0;
- //log directory
- //hadoop.log.dir TBD
+
dfs.info.bindAddress NULL;
dfs.info.port NULL;
dfs.http.address NULL;
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/conf/ConfigurationAttributes.java
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/conf/ConfigurationAttributes.java 2008-05-07 22:20:49 UTC (rev 6490)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/conf/ConfigurationAttributes.java 2008-05-07 22:33:34 UTC (rev 6491)
@@ -202,4 +202,8 @@
*/
String SESSION_ID = "session.id";
+ String DFS_DATA_DIR = "dfs.data.dir";
+ String DFS_NAME_DIR = "dfs.name.dir";
+ String DFS_NAMENODE_STARTUP = "dfs.namenode.startup";
+ String HADOOP_LOG_DIR = "hadoop.log.dir";
}
Modified: trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/hplb.sf
===================================================================
--- trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/hplb.sf 2008-05-07 22:20:49 UTC (rev 6490)
+++ trunk/core/components/hadoop/src/org/smartfrog/services/hadoop/examples/hplb.sf 2008-05-07 22:33:34 UTC (rev 6491)
@@ -36,7 +36,7 @@
namenode (nameNodeHost ++ ":" ++ namenodePort );
filesystem ("hdfs://" ++ namenode);
trackerUrl ("http:" ++ jobTracker ++ ":" ++ trackerWebPort ++ "/");
- nameUrl ("http:" ++ jobTracker ++ ":" ++ trackerWebPort ++ "/");
+ nameUrl ("http:" ++ jobTracker ++ ":" ++ nameWebPort ++ "/");
tmpDir "/home/hadoop/tmp/hadoop-hadoop";
user "hadoop";
group "supergroup";
Modified: trunk/core/components/hadoop/test/org/smartfrog/services/hadoop/test/system/local/components.sf
===================================================================
--- trunk/core/components/hadoop/test/org/smartfrog/services/hadoop/test/system/local/components.sf 2008-05-07 22:20:49 UTC (rev 6490)
+++ trunk/core/components/hadoop/test/org/smartfrog/services/hadoop/test/system/local/components.sf 2008-05-07 22:33:34 UTC (rev 6491)
@@ -23,9 +23,9 @@
LocalHadoopCompound extends Compound {
- fs.default.name "localhost:8020";
+ fs.default.name "hdfs://localhost:8020/";
- hadoop.log.dir extends TempDirWithCleanup {
+ logDir extends TempDirWithCleanup {
}
@@ -33,5 +33,8 @@
}
+ nameDir extends TempDirWithCleanup {
+ }
+
}
\ No newline at end of file
Modified: trunk/core/components/hadoop/test/org/smartfrog/services/hadoop/test/system/local/namenode/components.sf
===================================================================
--- trunk/core/components/hadoop/test/org/smartfrog/services/hadoop/test/system/local/namenode/components.sf 2008-05-07 22:20:49 UTC (rev 6490)
+++ trunk/core/components/hadoop/test/org/smartfrog/services/hadoop/test/system/local/namenode/components.sf 2008-05-07 22:33:34 UTC (rev 6491)
@@ -29,10 +29,13 @@
LocalNameNodeCompound extends LocalHadoopCompound {
LocalInProcessNameNode extends NameNode {
-
+ fs.default.name PARENT:fs.default.name;
dataDirectories [LAZY dataDir];
-
-
+ nameDirectories [LAZY nameDir];
+ logDir LAZY PARENT:logDir;
+ dfs.namenode.startup ACTION_FORMAT;
+ dfs.http.address "http://localhost:8021";
+ dfs.permissions false;
}
}
@@ -41,7 +44,8 @@
* Test the local name node
*/
testLocalNameNodeCompound extends ExpectDeploy {
- action extends LocalNameNodeCompound;
+ action extends LocalNameNodeCompound {
+ }
tests extends Sequence {
//TODO: some tests
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|