Menu
▾
▴
bigdata-commit
|
From: <tho...@us...> - 2013-11-22 15:49:56
|
Revision: 7582
http://bigdata.svn.sourceforge.net/bigdata/?rev=7582&view=rev
Author: thompsonbry
Date: 2013-11-22 15:49:49 +0000 (Fri, 22 Nov 2013)
Log Message:
-----------
Added ability to set the river locators and groups and the zookeeper servers from the environment outside of the startHAServices script.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config
branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config 2013-11-22 15:43:38 UTC (rev 7581)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config 2013-11-22 15:49:49 UTC (rev 7582)
@@ -116,7 +116,7 @@
//static private groups = LookupDiscovery.ALL_GROUPS;
// unicast discovery or multiple setups, MUST specify groups.
- static private groups = new String[]{bigdata.fedname};
+ static private groups = ConfigMath.getGroups(System.getProperty("GROUPS",bigdata.fedname));
/**
* One or more unicast URIs of the form <code>jini://host/</code>
@@ -126,15 +126,8 @@
* discovery <strong>and</strong> you have specified the groups as
* LookupDiscovery.ALL_GROUPS (a <code>null</code>).
*/
- static private locators = new LookupLocator[] {
-
- // runs jini on one or more hosts using unicast locators.
- new LookupLocator("jini://bigdata15/"),
- new LookupLocator("jini://bigdata16/"),
- new LookupLocator("jini://bigdata17/"),
-
- };
-
+ static private locators = ConfigMath.getLocators(System.getProperty("LOCATORS","jini://bigdata15/,jini://bigdata16/,jini://bigdata17/"));
+
/**
* A common point to set the Zookeeper client's requested
* sessionTimeout and the jini lease timeout. The default lease
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices 2013-11-22 15:43:38 UTC (rev 7581)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices 2013-11-22 15:49:49 UTC (rev 7582)
@@ -31,6 +31,13 @@
##
export JVM_OPTS="-server -Xmx4G -XX:MaxDirectMemorySize=3000m"
+# Apache River
+export GROUPS="$FEDNAME"
+export LOCATORS="jini://bigdata15/,jini://bigdata16/,jini://bigdata17/"
+
+# Apache Zookeeper
+export ZK_SERVERS="bigdata15:2081,bigdata16:2081,bigdata17:2081";
+
##
# HAJournalServer configuration parameter overrides (see HAJournal.config).
#
@@ -43,12 +50,23 @@
# Note: Many of these properties have defaults.
##
-export FEDNAME=installTest
-export LOGICAL_SERVICE_ID=HAJournalServer-1
-export FED_DIR=$INSTALL_DIR
-export REPLICATION_FACTOR=3
-export HA_PORT=9090
-export NSS_PORT=8080
+# Conditional defaults for required properties. These can (and should) be
+# overridden from the environment outside of this script. The defaults are
+# not suitable for deployment.
+if [ -z "${FEDNAME}" ]; then
+ export FEDNAME=installTest
+fi
+if [ -z "${LOGICAL_SERVICE_ID}" ]; then
+ export LOGICAL_SERVICE_ID=HAJournalServer-1
+fi
+if [ -z "${FED_DIR}" ]; then
+ export FED_DIR=$INSTALL_DIR
+fi
+
+# All of these have defaults.
+#export REPLICATION_FACTOR=3
+#export HA_PORT=9090
+#export NSS_PORT=8080
#export QUERY_THREAD_POOL_SIZE=
#export COLLECT_QUEUE_STATISTICS=
#export COLLECT_PLATFORM_STATISTICS=
@@ -56,11 +74,6 @@
#export GANGLIA_LISTENER=
#export SYSSTAT_DIR=
-#export GROUPS=
-#export LOCATORS=
-
-export ZK_SERVERS="bigdata15:2081,bigdata16:2081,bigdata17:2081";
-
export HAOPTS="\
-DFEDNAME=${FEDNAME}\
-DLOGICAL_SERVICE_ID=${LOGICAL_SERVICE_ID}\
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-04-04 14:48:26
|
Revision: 8043
http://sourceforge.net/p/bigdata/code/8043
Author: thompsonbry
Date: 2014-04-04 14:48:24 +0000 (Fri, 04 Apr 2014)
Log Message:
-----------
Define DATA_DIR as the location of the journal file as an environment variable that is passed through from startHAServices and used by HAJournal.config.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config
branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config 2014-04-03 23:39:45 UTC (rev 8042)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config 2014-04-04 14:48:24 UTC (rev 8043)
@@ -83,7 +83,7 @@
private static serviceDir = new File(fedDir,logicalServiceId+File.separator+"HAJournalServer");
// journal data directory.
- private static dataDir = serviceDir;
+ private static dataDir = new File(ConfigMath.getProperty("DATA_DIR",serviceDir.toString());
// HA log directory.
private static haLogDir = new File(serviceDir,"HALog");
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices 2014-04-03 23:39:45 UTC (rev 8042)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices 2014-04-04 14:48:24 UTC (rev 8043)
@@ -75,6 +75,7 @@
-DFEDNAME=${FEDNAME}\
-DLOGICAL_SERVICE_ID=${LOGICAL_SERVICE_ID}\
-DFED_DIR=${FED_DIR}\
+ -DDATA_DIR=${DATA_DIR}\
-DREPLICATION_FACTOR=${REPLICATION_FACTOR}\
-DHA_PORT=${HA_PORT}\
"-Djetty.port=${JETTY_PORT}"\
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <dme...@us...> - 2014-04-14 09:52:41
|
Revision: 8114
http://sourceforge.net/p/bigdata/code/8114
Author: dmekonnen
Date: 2014-04-14 09:52:38 +0000 (Mon, 14 Apr 2014)
Log Message:
-----------
updates to support running HARestore upon reboot.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/init.d/bigdataHA
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/HARestore
Added: branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/HARestore
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/HARestore (rev 0)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/HARestore 2014-04-14 09:52:38 UTC (rev 8114)
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+source /etc/default/bigdataHA
+
+SERVICE_DIR="$FED_DIR/$FEDNAME/$LOGICAL_SERVICE_ID/HAJournalServer"
+LIB_DIR="$FED_DIR/lib"
+
+java -cp $LIB_DIR/bigdata.jar:$LIB_DIR/commons-logging.jar:$LIB_DIR/log4j.jar:$LIB_DIR/highscalelib.jar:$LIB_DIR/fastutil.jar:$LIB_DIR/dsiutils.jar:$LIB_DIR/lgplutils.jar:$LIB_DIR/icu4j.jar -Dlog4j.configuration=file:var/config/logging/log4j.properties com.bigdata.journal.jini.ha.HARestore -o $DATA_DIR/bigdata-ha.jnl $SERVICE_DIR/snapshot $SERVICE_DIR/HALog
+
+
Property changes on: branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/HARestore
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/init.d/bigdataHA
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/init.d/bigdataHA 2014-04-13 04:05:32 UTC (rev 8113)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/init.d/bigdataHA 2014-04-14 09:52:38 UTC (rev 8114)
@@ -70,7 +70,13 @@
cd $FED_DIR
+if [ ! -d "$DATA_DIR/lost+found" ]; then
+ mount /dev/vg/lv_bigdata $DATA_DIR
+ action $"`date` : `hostname` : restoring bigdata journal file: " sudo -u $BD_USER -g $BD_GROUP bin/HARestore
+fi
+
+
#
# See how we were called.
#
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 13:56:32
|
Revision: 8309
http://sourceforge.net/p/bigdata/code/8309
Author: thompsonbry
Date: 2014-05-14 13:56:29 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Changed the name of the environment variable to enable the built-in ganglia peer as a listener from GANGLIA_LISTENER => GANGLIA_LISTEN
See #624 (HA LBS).
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config
branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices
branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config 2014-05-14 10:52:46 UTC (rev 8308)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config 2014-05-14 13:56:29 UTC (rev 8309)
@@ -355,15 +355,15 @@
// Platform and process performance counters. This requires external
// software on some platforms (vmstat, pidstat, iostat, etc.).
//
- // This is necessary for the GangliaLBSPolicy.
+ // This is necessary for the GangliaLBSPolicy or CountersLBSPolicy.
new NV(Journal.Options.COLLECT_PLATFORM_STATISTICS,
- ConfigMath.getProperty("COLLECT_PLATFORM_STATISTICS","true")),
+ ConfigMath.getProperty("COLLECT_PLATFORM_STATISTICS","false")),
// Use bigdata-ganglia module to build internal model of cluster load.
//
// This is required for the GangliaLBSPolicy.
new NV(com.bigdata.journal.GangliaPlugIn.Options.GANGLIA_LISTEN,
- ConfigMath.getProperty("GANGLIA_LISTENER","true")),
+ ConfigMath.getProperty("GANGLIA_LISTEN","false")),
// Use bigdata-ganglia module to report service metrics to ganglia.
//
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices 2014-05-14 10:52:46 UTC (rev 8308)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices 2014-05-14 13:56:29 UTC (rev 8309)
@@ -89,6 +89,7 @@
-DCOLLECT_QUEUE_STATISTICS=${COLLECT_QUEUE_STATISTICS}\
-DCOLLECT_PLATFORM_STATISTICS=${COLLECT_PLATFORM_STATISTICS}\
-DGANGLIA_REPORT=${GANGLIA_REPORT}\
+ -DGANGLIA_LISTEN=${GANGLIA_LISTEN}\
-DSYSSTAT_DIR=${SYSSTAT_DIR}\
-Dcom.bigdata.counters.linux.sysstat.path=${SYSSTAT_DIR}\
"
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA 2014-05-14 10:52:46 UTC (rev 8308)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA 2014-05-14 13:56:29 UTC (rev 8309)
@@ -46,5 +46,5 @@
#export COLLECT_QUEUE_STATISTICS=
#export COLLECT_PLATFORM_STATISTICS=
#export GANGLIA_REPORT=
-#export GANGLIA_LISTENER=
+#export GANGLIA_LISTEN=
#export SYSSTAT_DIR=
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-06-09 19:30:32
|
Revision: 8455
http://sourceforge.net/p/bigdata/code/8455
Author: thompsonbry
Date: 2014-06-09 19:30:26 +0000 (Mon, 09 Jun 2014)
Log Message:
-----------
Added an HALOG_DIR and a SNAPSHOT_DIR to allow these things to be set independently of the DATA_DIR (which is where the journal lives).
Updated the wiki page at http://wiki.bigdata.com/wiki/index.php/HAJournalServer#Durable_Data to declare these new variables and document others.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config
branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config 2014-06-09 17:42:45 UTC (rev 8454)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config 2014-06-09 19:30:26 UTC (rev 8455)
@@ -89,10 +89,12 @@
private static dataDir = new File(ConfigMath.getProperty("DATA_DIR",""+serviceDir));
// HA log directory.
- private static haLogDir = new File(serviceDir,"HALog");
+ private static haLogDir = new File(ConfigMath.getProperty("HALOG_DIR",""+serviceDir+File.separator+"HALog"));
+ //private static haLogDir = new File(serviceDir,"HALog");
// Snapshot directory.
- private static snapshotDir = new File(serviceDir,"snapshot");
+ private static snapshot = new File(ConfigMath.getProperty("SNAPSHOT_DIR",""+serviceDir+File.separator+"snapshot"));
+ //private static snapshotDir = new File(serviceDir,"snapshot");
/* Snapshot policy. Choose one.
*
@@ -332,7 +334,7 @@
new NV(Options.BUFFER_MODE,""+BufferMode.DiskRW),
- new NV(Options.WRITE_CACHE_BUFFER_COUNT,"2000"),
+ new NV(Options.WRITE_CACHE_BUFFER_COUNT,ConfigMath.getProperty("WRITE_CACHE_BUFFER_COUNT","2000")),
new NV(IndexMetadata.Options.WRITE_RETENTION_QUEUE_CAPACITY,"4000"),
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA 2014-06-09 17:42:45 UTC (rev 8454)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA 2014-06-09 19:30:26 UTC (rev 8455)
@@ -55,6 +55,10 @@
#export JETTY_PORT=8080
#export JETTY_XML=var/jetty/WEB-INF/jetty.xml
#export JETTY_RESOURCE_BASE=var/jetty
+#export WRITE_CACHE_BUFFER_COUNT=2000
+#export DATA_DIR=
+#export HALOG_DIR=
+#export SNAPSHOT_DIR=
#export COLLECT_QUEUE_STATISTICS=
#export COLLECT_PLATFORM_STATISTICS=
#export GANGLIA_REPORT=
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-06-17 13:29:53
|
Revision: 8495
http://sourceforge.net/p/bigdata/code/8495
Author: thompsonbry
Date: 2014-06-17 13:29:50 +0000 (Tue, 17 Jun 2014)
Log Message:
-----------
HA service start changes:
- startHAServices: Now requires explicit configuration of LOCATORS and ZK_SERVERS. Now passes through the WRITE_CACHE_BUFFER_COUNT environment variable. This is used by the HAJournal.config.
- bigdataHA: Now specifies JVM_OPTS IFF they are not specified in the environment. This allows them to be overridden by the caller. If you want to have the defaults, then "unset" the JVM_OPTS variable in the environment before invoking this script.
See #965 (HA1 LBS)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices
branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices 2014-06-17 13:28:07 UTC (rev 8494)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices 2014-06-17 13:29:50 UTC (rev 8495)
@@ -54,10 +54,14 @@
export GROUPS="$FEDNAME"
fi
if [ -z "${LOCATORS}" ]; then
- export LOCATORS="jini://bigdata15/,jini://bigdata16/,jini://bigdata17/"
+# export LOCATORS="jini://bigdata15/,jini://bigdata16/,jini://bigdata17/"
+ echo "Must specify: LOCATORS"
+ exit 1
fi
if [ -z "${ZK_SERVERS}" ]; then
- export ZK_SERVERS="bigdata15:2081,bigdata16:2081,bigdata17:2081";
+# export ZK_SERVERS="bigdata15:2081,bigdata16:2081,bigdata17:2081"
+ echo "Must specify: ZK_SERVERS"
+ exit 1
fi
export HA_OPTS="\
@@ -71,6 +75,7 @@
-DZK_SERVERS=${ZK_SERVERS}\
-DRMI_PORT=${RMI_PORT}\
-DHA_PORT=${HA_PORT}\
+ -DWRITE_CACHE_BUFFER_COUNT=${WRITE_CACHE_BUFFER_COUNT}\
"-Dcom.bigdata.hostname=${BIGDATA_HOSTNAME}"\
"-Djetty.port=${JETTY_PORT}"\
"-Djetty.threads.min=${JETTY_THREADS_MIN}"\
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA 2014-06-17 13:28:07 UTC (rev 8494)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA 2014-06-17 13:29:50 UTC (rev 8495)
@@ -22,8 +22,13 @@
# size of the direct memory heap (used for the write cache buffers and
# some related things).
##
-export JVM_OPTS="-server -Xmx4G -XX:MaxDirectMemorySize=3000m"
-#export JVM_OPTS="-server -Xmx4G -XX:MaxDirectMemorySize=3000m -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=1046"
+if [ -z "$JVM_OPTS" ]; then
+ export JVM_OPTS="-server -Xmx4G -XX:MaxDirectMemorySize=3000m"
+# export JVM_OPTS="-server -Xmx4G -XX:MaxDirectMemorySize=3000m -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=1046"
+ echo "Set JVM_OPTS=${JVM_OPTS}"
+else
+ echo "Using JVM_OPTS=${JVM_OPTS}"
+fi
##
# The following variables configure the startHAServices script, which
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
×
Want the latest updates on software, tech news, and AI?
Get latest updates about software, tech news, and AI from SourceForge directly in your inbox once a month.
Thanks for helping keep SourceForge clean.
X