Menu
▾
▴
[Smartfrog-checkins] SF.net SVN: smartfrog:[8715] trunk/core/hadoop-components/hadoop-ops
|
From: <st...@us...> - 2011-07-27 15:18:35
|
Revision: 8715
http://smartfrog.svn.sourceforge.net/smartfrog/?rev=8715&view=rev
Author: steve_l
Date: 2011-07-27 15:18:29 +0000 (Wed, 27 Jul 2011)
Log Message:
-----------
SFOS-1641 ManagedConfiguration to (optionally) propagate changes up to the bound component
Modified Paths:
--------------
trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/components/submitter/SubmitterImpl.java
trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/components/submitter/ToolRunnerComponentImpl.java
trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/ConfigurationLoader.java
trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/HadoopConfiguration.java
trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/ManagedConfiguration.java
trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/WriteClusterConfigImpl.java
trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/core/HadoopComponentImpl.java
trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/dfs/DfsClusterBoundImpl.java
trunk/core/hadoop-components/hadoop-ops/test/org/smartfrog/services/hadoop/operations/test/system/conf/ManagedConfTestPrimImpl.java
trunk/core/hadoop-components/hadoop-ops/test/org/smartfrog/services/hadoop/operations/test/system/conf/components.sf
Modified: trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/components/submitter/SubmitterImpl.java
===================================================================
--- trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/components/submitter/SubmitterImpl.java 2011-07-26 16:33:09 UTC (rev 8714)
+++ trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/components/submitter/SubmitterImpl.java 2011-07-27 15:18:29 UTC (rev 8715)
@@ -94,7 +94,7 @@
dumpOnFailure = sfResolve(ATTR_DUMP_ON_FAILURE, false, true);
//create the job configuration. The cluster reference is optional
- jobConf = ManagedConfiguration.createConfiguration(this, true, false, true);
+ jobConf = ManagedConfiguration.createConfiguration(this, true, false, true, false);
//look for the file
boolean fileRequired = sfResolve(ATTR_FILE_REQUIRED, true, true);
Modified: trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/components/submitter/ToolRunnerComponentImpl.java
===================================================================
--- trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/components/submitter/ToolRunnerComponentImpl.java 2011-07-26 16:33:09 UTC (rev 8714)
+++ trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/components/submitter/ToolRunnerComponentImpl.java 2011-07-27 15:18:29 UTC (rev 8715)
@@ -90,7 +90,7 @@
+ " cannot be instantiated: " + e, e);
}
//sort out the configuration
- toolConf = ManagedConfiguration.createConfiguration(this, true, false, true);
+ toolConf = ManagedConfiguration.createConfiguration(this, true, false, true, false);
//start the tool in a new thread
worker = new WorkflowThread(this, this, true);
worker.start();
Modified: trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/ConfigurationLoader.java
===================================================================
--- trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/ConfigurationLoader.java 2011-07-26 16:33:09 UTC (rev 8714)
+++ trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/ConfigurationLoader.java 2011-07-27 15:18:29 UTC (rev 8715)
@@ -59,6 +59,7 @@
if (debug) {
source.sfLog().debug("Load default resources=" + loadDefaults);
}
+ boolean propagateReloads = source.sfResolve(HadoopConfiguration.ATTR_PROPAGATE_RELOADS, false, false);
List<String> resources = ListUtils
.resolveStringList(source, new Reference(HadoopConfiguration.ATTR_RESOURCES), false);
if (resources == null) {
@@ -78,7 +79,7 @@
if (!resources.isEmpty() || !files.isEmpty()) {
throw new SmartFrogResolutionException(ERROR_NO_CLUSTER_AND_XML);
}
- conf = ManagedConfiguration.createConfiguration(source, true, false, loadDefaults);
+ conf = ManagedConfiguration.createConfiguration(source, true, false, loadDefaults, propagateReloads);
conf.copyPropertiesToPrim(source);
} else {
Modified: trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/HadoopConfiguration.java
===================================================================
--- trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/HadoopConfiguration.java 2011-07-26 16:33:09 UTC (rev 8714)
+++ trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/HadoopConfiguration.java 2011-07-27 15:18:29 UTC (rev 8715)
@@ -35,4 +35,5 @@
String ATTR_READ_EARLY = "conf.read.early";
String ATTR_DUMP = "conf.dump";
String ATTR_REQUIRED = "conf.required";
+ String ATTR_PROPAGATE_RELOADS = "conf.propagate.reloads";
}
Modified: trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/ManagedConfiguration.java
===================================================================
--- trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/ManagedConfiguration.java 2011-07-26 16:33:09 UTC (rev 8714)
+++ trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/ManagedConfiguration.java 2011-07-27 15:18:29 UTC (rev 8715)
@@ -62,16 +62,17 @@
public final class ManagedConfiguration extends JobConf implements PrimSource,
ConfigurationAttributes, Cloneable {
- private static final Log Log = LogFactory.getLog(ManagedConfiguration.class);
+ private static final Log LOG = LogFactory.getLog(ManagedConfiguration.class);
private Prim source;
-
+ private String description;
+ boolean propagateReloads;
+
/*
* Force load the extra configurations
*/
static {
ConfigurationLoader.loadExtendedConfigurations();
}
-
/**
* Some attributes that are not listed in the component (so they can be picked up from parents) but which should be
* discovered.
@@ -82,7 +83,6 @@
MAPRED_LOCAL_DIR
};*/
public static final String MISSING_ATTRIBUTE = "Missing attribute";
- private String description;
private static final Reference REF_CLUSTER = new Reference(ClusterBound.ATTR_CLUSTER);
/**
@@ -101,22 +101,41 @@
}
}*/
+
+
/**
* A new map/reduce configuration where the behavior of reading from the default resources can be turned off. <p/>
* If the parameter {@code loadDefaults} is false, the new instance will not load resources from the default files.
*
* @param loadDefaults specifies whether to load from the default files
* @param source source of config information
+ * @param propagateReloads should reloads be passed up
* @throws RemoteException for network problems
* @throws SmartFrogResolutionException for resolution problems
*/
- public ManagedConfiguration(boolean loadDefaults, Prim source) throws SmartFrogException,
+ public ManagedConfiguration(boolean loadDefaults, Prim source, boolean propagateReloads) throws SmartFrogException,
RemoteException {
super(loadDefaults);
+ this.propagateReloads = propagateReloads;
bind(source);
}
/**
+ * A new map/reduce configuration where the behavior of reading from the default resources can be turned off. <p/>
+ * If the parameter {@code loadDefaults} is false, the new instance will not load resources from the default files.
+ *
+ * @param loadDefaults specifies whether to load from the default files
+ * @param source source of config information
+ * @throws RemoteException for network problems
+ * @throws SmartFrogResolutionException for resolution problems
+ */
+ public ManagedConfiguration(boolean loadDefaults, Prim source) throws SmartFrogException,
+ RemoteException {
+ this(loadDefaults, source, false);
+ }
+
+
+ /**
* A new configuration. Default values are picked up
*
* @param source source of config information
@@ -656,11 +675,13 @@
/**
* This creates a configuration from a source prim
*
+ *
* @param source source prim
* @param useClusterReference if set, resolve {@link ClusterBound#ATTR_CLUSTER} from the source and use that
* first.
* @param clusterRequired if set, the cluster attribute must resolve.
* @param loadDefaults flag to say "load the default values"
+ * @param propagateReloads flag to say "propagate any reloads back to the owner"
* @return the new element
* @throws SmartFrogException for any failure to resolve all the attributes
* @throws RemoteException network problems. These are always passed up
@@ -668,7 +689,7 @@
public static ManagedConfiguration createConfiguration(Prim source,
boolean useClusterReference,
boolean clusterRequired,
- boolean loadDefaults)
+ boolean loadDefaults, final boolean propagateReloads)
throws SmartFrogException, RemoteException {
ManagedConfiguration conf = new ManagedConfiguration(loadDefaults, source);
if (useClusterReference) {
@@ -684,10 +705,20 @@
}
+ /**
+ * push out changes
+ */
@Override
public void reloadConfiguration() {
- Log.info("Reloading configuration");
+ LOG.info("Reloading configuration under " + description);
super.reloadConfiguration();
+ if (propagateReloads) try {
+ copyComponentState(source, null);
+ } catch (RemoteException e) {
+ LOG.warn("When propagating configuration " + e, e);
+ } catch (SmartFrogResolutionException e) {
+ LOG.warn("When propagating configuration " + e, e);
+ }
}
public static void addNewDefaultResource(String resourceName) throws SmartFrogException {
@@ -695,7 +726,7 @@
if (url == null) {
throw new SmartFrogException("No resource \"" + resourceName + "\" found in the Configuration classpath");
}
- Log.info("Adding a new default resource \"" + resourceName + "\" from " + url);
+ LOG.info("Adding a new default resource \"" + resourceName + "\" from " + url);
addDefaultResource(resourceName);
}
}
Modified: trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/WriteClusterConfigImpl.java
===================================================================
--- trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/WriteClusterConfigImpl.java 2011-07-26 16:33:09 UTC (rev 8714)
+++ trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/conf/WriteClusterConfigImpl.java 2011-07-27 15:18:29 UTC (rev 8715)
@@ -25,7 +25,7 @@
@Override
public void sfStart() throws SmartFrogException, RemoteException {
super.sfStart();
- ManagedConfiguration configuration = ManagedConfiguration.createConfiguration(this, true, true, false);
+ ManagedConfiguration configuration = ManagedConfiguration.createConfiguration(this, true, true, false, false);
File destFile = FileSystem.lookupAbsoluteFile(this, FileUsingComponent.ATTR_FILENAME, null, null, true, null);
FileOutputStream fos = null;
try {
Modified: trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/core/HadoopComponentImpl.java
===================================================================
--- trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/core/HadoopComponentImpl.java 2011-07-26 16:33:09 UTC (rev 8714)
+++ trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/core/HadoopComponentImpl.java 2011-07-27 15:18:29 UTC (rev 8715)
@@ -91,7 +91,7 @@
*/
protected ManagedConfiguration createClusterAttrConfiguration()
throws SmartFrogException, RemoteException {
- return ManagedConfiguration.createConfiguration(this, true, true, false);
+ return ManagedConfiguration.createConfiguration(this, true, true, false, false);
}
/**
Modified: trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/dfs/DfsClusterBoundImpl.java
===================================================================
--- trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/dfs/DfsClusterBoundImpl.java 2011-07-26 16:33:09 UTC (rev 8714)
+++ trunk/core/hadoop-components/hadoop-ops/src/org/smartfrog/services/hadoop/operations/dfs/DfsClusterBoundImpl.java 2011-07-27 15:18:29 UTC (rev 8715)
@@ -92,7 +92,7 @@
* @throws RemoteException network problems
*/
public ManagedConfiguration createConfiguration() throws SmartFrogException, RemoteException {
- return ManagedConfiguration.createConfiguration(this, true, isClusterRequired(), false);
+ return ManagedConfiguration.createConfiguration(this, true, isClusterRequired(), false, false);
}
/**
Modified: trunk/core/hadoop-components/hadoop-ops/test/org/smartfrog/services/hadoop/operations/test/system/conf/ManagedConfTestPrimImpl.java
===================================================================
--- trunk/core/hadoop-components/hadoop-ops/test/org/smartfrog/services/hadoop/operations/test/system/conf/ManagedConfTestPrimImpl.java 2011-07-26 16:33:09 UTC (rev 8714)
+++ trunk/core/hadoop-components/hadoop-ops/test/org/smartfrog/services/hadoop/operations/test/system/conf/ManagedConfTestPrimImpl.java 2011-07-27 15:18:29 UTC (rev 8715)
@@ -65,7 +65,7 @@
dump = conf.dump();
String value1 = conf.get(P1, null);
assertTrue(P1 + "==" + value1 + " in\n" + dump, value1 != null);
- assertTrue(P1 + "!=" + V1 + "is is " + value1 + " in\n" + dump, V1.equals(value1));
+ assertTrue(P1 + "!=" + V1 + " -it is " + value1 + " in\n" + dump, V1.equals(value1));
conf.set(P2, V2);
dump = conf.dump();
String value2 = conf.get(P2, null);
Modified: trunk/core/hadoop-components/hadoop-ops/test/org/smartfrog/services/hadoop/operations/test/system/conf/components.sf
===================================================================
--- trunk/core/hadoop-components/hadoop-ops/test/org/smartfrog/services/hadoop/operations/test/system/conf/components.sf 2011-07-26 16:33:09 UTC (rev 8714)
+++ trunk/core/hadoop-components/hadoop-ops/test/org/smartfrog/services/hadoop/operations/test/system/conf/components.sf 2011-07-27 15:18:29 UTC (rev 8715)
@@ -212,17 +212,11 @@
}
// to handle race conditions between the add default working and the tests
- sleep extends Delay {
- time 1000;
- action extends DefaultHadoopConfiguration {
-
- }
+ config extends DefaultHadoopConfiguration {
+ conf.propagate.reloads true;
+ }
}
-
- config LAZY sleep:action;
- }
-
tests extends Sequence {
test.hdfs.uri extends AssertWorkflow {
@@ -237,9 +231,9 @@
}
test.value.one.set extends AssertWorkflow {
- equalityIgnoresCase true;
- equalsString1 "1";
- equalsString2 LAZY action:config:test.value.one;
+ reference LAZY action:config;
+ attribute "test.value.one";
+ attributeEquals "1";
}
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|