From: <dme...@us...> - 2014-05-06 17:57:50
|
Revision: 8208 http://sourceforge.net/p/bigdata/code/8208 Author: dmekonnen Date: 2014-05-06 17:57:47 +0000 (Tue, 06 May 2014) Log Message: ----------- Updates to sync with new jetty.xml and other updates through April. EC2 instance reboots can now be avoided (verification TBD). Modified Paths: -------------- branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/Berksfile branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/Vagrantfile branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/attributes/default.rb branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/bin/setHosts.py branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/metadata.rb branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/templates/default/default/bigdataHA.erb branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/templates/default/jetty.xml.erb branches/DEPLOYMENT_BRANCH_1_3_1/src/resources/etc/init.d/bigdataHA Removed Paths: ------------- branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/templates/default/init.d/bigdataHA.erb Modified: branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/Berksfile =================================================================== --- branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/Berksfile 2014-05-06 13:54:41 UTC (rev 8207) +++ branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/Berksfile 2014-05-06 17:57:47 UTC (rev 8208) @@ -7,6 +7,6 @@ cookbook 'lvm' cookbook "hadoop" cookbook "emacs" -# cookbook "ganglia" +cookbook "sysstat" metadata Modified: branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/Vagrantfile =================================================================== --- branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/Vagrantfile 2014-05-06 13:54:41 UTC (rev 8207) +++ branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/Vagrantfile 2014-05-06 17:57:47 UTC (rev 8208) @@ -63,6 +63,7 @@ "recipe[apt]", "recipe[java]", "recipe[emacs]", # default is no x11 + "recipe[sysstat]", # # use this recipe only if the oracle server is timing out in the above 'java' recipe # @@ -135,6 +136,7 @@ "recipe[apt]", "recipe[java]", "recipe[emacs]", # default is no x11 + "recipe[sysstat]", # # use this recipe only if the oracle server is timing out in the above 'java' recipe # @@ -207,6 +209,7 @@ "recipe[apt]", "recipe[java]", "recipe[emacs]", # default is no x11 + "recipe[sysstat]", # # use this recipe only if the oracle server is timing out in the above 'java' recipe # Modified: branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/attributes/default.rb =================================================================== --- branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/attributes/default.rb 2014-05-06 13:54:41 UTC (rev 8207) +++ branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/attributes/default.rb 2014-05-06 17:57:47 UTC (rev 8208) @@ -16,7 +16,7 @@ default['systap-bigdataHA'][:data_dir] = node['systap-bigdataHA'][:fed_dir] + "/data" # Where the log files will live: -default['systap-bigdataHA'][:log_dir] = node['systap-bigdataHA'][:fed_dir] + "/logs" +default['systap-bigdataHA'][:log_dir] = node['systap-bigdataHA'][:fed_dir] + "/log" # Name of the replication cluster to which this HAJournalServer will belong. default['systap-bigdataHA'][:logical_service_id] = 'HA-Replication-Cluster-1' Modified: branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/bin/setHosts.py =================================================================== --- branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/bin/setHosts.py 2014-05-06 13:54:41 UTC (rev 8207) +++ branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/bin/setHosts.py 2014-05-06 17:57:47 UTC (rev 8208) @@ -35,8 +35,12 @@ def createZookeeperSubstitution( index, host, ipAddress ): return "sudo sed -i 's|server." + index + "=" + host + "|server." + index + "=" + ipAddress + "|' /etc/zookeeper/conf/zoo.cfg" -def createBigdataHASubstitution( host, ipAddress ): - return "sudo sed -i 's|" + host + "|" + ipAddress + "|' /etc/defaults/bigdataHA" +def createJiniLocatorsSubstitution(): + locators = "" + for host in hostMap: + locators = locators + "jini://" + hostMap[host] + "/," + locators = locators[:-1] + return "sudo sed -i 's|%JINI_LOCATORS%|" + locators + "|' /etc/default/bigdataHA" if __name__ == '__main__': @@ -55,6 +59,9 @@ private_security_group_name = os.environ["AWS_SECURITY_GROUP_PRIVATE"] group = ec2conn.get_all_security_groups( private_security_group_name )[0] + jini_locators = createJiniLocatorsSubstitution() + # print "JINI_LOCATORS = " + jini_locators + i = 1 for host in bigdataHosts: ssh_client = sshclient_from_instance( host, key_path, user_name='ubuntu' ) @@ -70,11 +77,9 @@ status, stdin, stderr = ssh_client.run( createZookeeperSubstitution( "2", bigdataB, hostMap[ bigdataB ] ) ) status, stdin, stderr = ssh_client.run( createZookeeperSubstitution( "3", bigdataC, hostMap[ bigdataC ] ) ) - name = host.__dict__['tags']['Name'] - hostAddress = host.__dict__['private_ip_address'] - # status, stdin, stderr = ssh_client.run( createBigdataHASubstitution( name, hostAddress ) ) + status, stdin, stderr = ssh_client.run( jini_locators ) - hostAddress = hostAddress + "/32" + hostAddress = host.__dict__['private_ip_address'] + "/32" group.authorize( ip_protocol="tcp", from_port="0", to_port="65535", cidr_ip=hostAddress, src_group=None ) i += 1 @@ -88,4 +93,4 @@ print "The hosts are now rebooting, this may take several minutes. \nOnce back up, you may confirm status by visiting:\n" for host in bigdataHosts: - print "\thttp://" + host.__dict__['ip_address'] + ":8080/status\n" + print "\thttp://" + host.__dict__['ip_address'] + ":8080/bigdata/status\n" Modified: branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/metadata.rb =================================================================== --- branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/metadata.rb 2014-05-06 13:54:41 UTC (rev 8207) +++ branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/metadata.rb 2014-05-06 17:57:47 UTC (rev 8208) @@ -13,3 +13,4 @@ depends 'lvm' depends 'hadoop' depends 'emacs' +depends 'sysstat' Modified: branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/templates/default/default/bigdataHA.erb =================================================================== --- branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/templates/default/default/bigdataHA.erb 2014-05-06 13:54:41 UTC (rev 8207) +++ branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/templates/default/default/bigdataHA.erb 2014-05-06 17:57:47 UTC (rev 8208) @@ -32,7 +32,7 @@ # Apache River - NO default for "LOCATORS". export GROUPS="${FEDNAME}" -export LOCATORS="jini://<%= node['systap-bigdataHA'][:river_locator1] %>/,jini://<%= node['systap-bigdataHA'][:river_locator2] %>/,jini://<%= node['systap-bigdataHA'][:river_locator3] %>/" +export LOCATORS="%JINI_LOCATORS%" # Apache ZooKeeper - NO default. export ZK_SERVERS="<%= node['systap-bigdataHA'][:zk_server1] %>:<%= node['zookeeper'][:zoocfg][:clientPort] %>,<%= node['systap-bigdataHA'][:zk_server2] %>:<%= node['zookeeper'][:zoocfg][:clientPort] %>,<%= node['systap-bigdataHA'][:zk_server3] %>:<%= node['zookeeper'][:zoocfg][:clientPort] %>" Deleted: branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/templates/default/init.d/bigdataHA.erb =================================================================== --- branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/templates/default/init.d/bigdataHA.erb 2014-05-06 13:54:41 UTC (rev 8207) +++ branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/templates/default/init.d/bigdataHA.erb 2014-05-06 17:57:47 UTC (rev 8208) @@ -1,133 +0,0 @@ -#!/bin/bash - -# init.d style script for bigdata HA services. The script can be used -# to 'start' or 'stop' services. -# -# Environment: -# -# binDir - The directory containing the installed scripts. -# pidFile - The pid is written on this file. -# -# Misc. -# -# See http://tldp.org/LDP/abs/html/index.html -# -# Note: Blank lines are significant in shell scripts. -# -# Note: Children must do "exit 0" to indicate success. -# -# Note: Convert DOS cr-lf to unix style in emacs: C-x RET f then unix - -# Source function library (just used for 'action'). If you don't have this -# it SHOULD automatically use the inline definition for "action()". -if [ -f "/etc/init.d/functions" ]; then - . /etc/init.d/functions -else -# Run some action. Log its output. No fancy colors. First argument is the -# label for the log file. Remaining arguments are the command to execute -# and its arguments, if any. - action() { - local STRING rc - STRING=$1 - echo -n "$STRING " - shift - sudo -u bigdata -g bigdata "$@" && echo -n "[OK]" || echo -n "[FAILED]" - rc=$? - echo - return $rc - } -fi - -# Where the scripts live. -cd `dirname $0` - -## -# Highly Recommended OS Tuning. -## - -# Do not swap out applications while there is free memory. -#/sbin/sysctl -w vm.swappiness=0 - -# Setup the environment. -source /etc/default/bigdataHA - -if [ -z "$binDir" ]; then - echo $"$0 : environment not setup: binDir is undefined." - exit 1; -fi -if [ -z "$pidFile" ]; then - echo $"$0 : environment not setup: pidFile is undefined" - exit 1; -fi - -# -# See how we were called. -# -case "$1" in - start) - - cd <%= node['systap-bigdataHA'][:fed_dir] %> -# -# Start the ServiceStarter and child services if not running. -# - if [ -f "$pidFile" ]; then - read pid < "$pidFile" - pidno=$( ps ax | grep $pid | awk '{ print $1 }' | grep $pid ) - if [ -z "$pidno" ]; then -# The process has died so remove the old pid file. - echo $"`date` : `hostname` : $pid died?" - rm -f "$pidFile" - fi - fi - if [ ! -f "$pidFile" ]; then - action $"`date` : `hostname` : bringing up services: " $binDir/startHAServices - else - echo $"`date` : `hostname` : running as $pid" - fi - ;; - stop) -# -# Stop the ServiceStarter and all child services. -# - if [ -f "$pidFile" ]; then - read pid < "$pidFile" - pidno=$( ps ax | grep $pid | awk '{ print $1 }' | grep $pid ) - if [ -z "$pidno" ]; then -# The process has died so remove the old pid file. - echo $"`date` : `hostname` : $pid died?" - rm -f "$pidFile" - else - action $"`date` : `hostname` : bringing down services: " kill $pid - rm -f "$pidFile" - fi - fi - ;; - status) -# -# Report status for the ServicesManager (up or down). -# - if [ -f "$pidFile" ]; then - read pid < "$pidFile" - pidno=$( ps ax | grep $pid | awk '{ print $1 }' | grep $pid ) - if [ -z "$pidno" ]; then - echo $"`date` : `hostname` : process died? pid=$pid." - else - echo $"`date` : `hostname` : running as $pid." - fi - else - echo $"`date` : `hostname` : not running." - fi - ;; - restart) - $0 stop - $0 start - ;; - *) -# -# Usage -# - echo $"Usage: $0 {start|stop|status|restart}" - exit 1 -esac - -exit 0 Modified: branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/templates/default/jetty.xml.erb =================================================================== --- branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/templates/default/jetty.xml.erb 2014-05-06 13:54:41 UTC (rev 8207) +++ branches/DEPLOYMENT_BRANCH_1_3_1/bigdata/src/resources/deployment/vagrant/systap-aws-bigdata-ha/templates/default/jetty.xml.erb 2014-05-06 17:57:47 UTC (rev 8208) @@ -5,29 +5,119 @@ <Configure id="Server" class="org.eclipse.jetty.server.Server"> <!-- =========================================================== --> - <!-- Server Thread Pool --> + <!-- Configure the Server Thread Pool. --> + <!-- The server holds a common thread pool which is used by --> + <!-- default as the executor used by all connectors and servlet --> + <!-- dispatches. --> + <!-- --> + <!-- Configuring a fixed thread pool is vital to controlling the --> + <!-- maximal memory footprint of the server and is a key tuning --> + <!-- parameter for tuning. In an application that rarely blocks --> + <!-- then maximal threads may be close to the number of 5*CPUs. --> + <!-- In an application that frequently blocks, then maximal --> + <!-- threads should be set as high as possible given the memory --> + <!-- available. --> + <!-- --> + <!-- Consult the javadoc of o.e.j.util.thread.QueuedThreadPool --> + <!-- for all configuration that may be set here. --> <!-- =========================================================== --> - <Set name="ThreadPool"> - <!-- Default queued blocking threadpool --> - <New class="org.eclipse.jetty.util.thread.QueuedThreadPool"> - <Set name="minThreads">10</Set> - <Set name="maxThreads">64</Set> - </New> - </Set> + <!-- uncomment to change type of threadpool --> + <Arg name="threadpool"><New id="threadpool" class="org.eclipse.jetty.util.thread.QueuedThreadPool"/></Arg> + <!-- --> + <Get name="ThreadPool"> + <Set name="minThreads" type="int"><SystemProperty name="jetty.threads.min" default="10"/></Set> + <Set name="maxThreads" type="int"><SystemProperty name="jetty.threads.max" default="64"/></Set> + <Set name="idleTimeout" type="int"><SystemProperty name="jetty.threads.timeout" default="60000"/></Set> + <Set name="detailedDump">false</Set> + </Get> + <!-- =========================================================== --> + <!-- Get the platform mbean server --> + <!-- =========================================================== --> + <Call id="MBeanServer" class="java.lang.management.ManagementFactory" + name="getPlatformMBeanServer" /> + + <!-- =========================================================== --> + <!-- Initialize the Jetty MBean container --> + <!-- =========================================================== --> + <Call name="addBean"> + <Arg> + <New id="MBeanContainer" class="org.eclipse.jetty.jmx.MBeanContainer"> + <Arg> + <Ref refid="MBeanServer" /> + </Arg> + </New> + </Arg> + </Call> + + <!-- Add the static log to the MBean server. --> + <Call name="addBean"> + <Arg> + <New class="org.eclipse.jetty.util.log.Log" /> + </Arg> + </Call> + + <!-- For remote MBean access (optional) --> + <New id="ConnectorServer" class="org.eclipse.jetty.jmx.ConnectorServer"> + <Arg> + <New class="javax.management.remote.JMXServiceURL"> + <Arg type="java.lang.String">rmi</Arg> + <Arg type="java.lang.String" /> + <Arg type="java.lang.Integer"><SystemProperty name="jetty.jmxrmiport" default="1090"/></Arg> + <Arg type="java.lang.String">/jndi/rmi://<SystemProperty name="jetty.jmxrmihost" default="localhost"/>:<SystemProperty name="jetty.jmxrmiport" default="1099"/>/jmxrmi</Arg> + </New> + </Arg> + <Arg>org.eclipse.jetty.jmx:name=rmiconnectorserver</Arg> + <Call name="start" /> + </New> + <!-- =========================================================== --> - <!-- Set connectors --> + <!-- Http Configuration. --> + <!-- This is a common configuration instance used by all --> + <!-- connectors that can carry HTTP semantics (HTTP, HTTPS, SPDY)--> + <!-- It configures the non wire protocol aspects of the HTTP --> + <!-- semantic. --> + <!-- --> + <!-- Consult the javadoc of o.e.j.server.HttpConfiguration --> + <!-- for all configuration that may be set here. --> <!-- =========================================================== --> + <New id="httpConfig" class="org.eclipse.jetty.server.HttpConfiguration"> + <Set name="secureScheme">https</Set> + <Set name="securePort"><Property name="jetty.secure.port" default="8443" /></Set> + <Set name="outputBufferSize"><Property name="jetty.output.buffer.size" default="32768" /></Set> + <Set name="requestHeaderSize"><Property name="jetty.request.header.size" default="8192" /></Set> + <Set name="responseHeaderSize"><Property name="jetty.response.header.size" default="8192" /></Set> + <Set name="sendServerVersion"><Property name="jetty.send.server.version" default="true" /></Set> + <Set name="sendDateHeader"><Property name="jetty.send.date.header" default="false" /></Set> + <Set name="headerCacheSize">512</Set> + <!-- Uncomment to enable handling of X-Forwarded- style headers + <Call name="addCustomizer"> + <Arg><New class="org.eclipse.jetty.server.ForwardedRequestCustomizer"/></Arg> + </Call> + --> + </New> + + <!-- Configure the HTTP endpoint. --> + <Call name="addConnector"> + <Arg> + <New class="org.eclipse.jetty.server.ServerConnector"> + <Arg name="server"><Ref refid="Server" /></Arg> + <Arg name="factories"> + <Array type="org.eclipse.jetty.server.ConnectionFactory"> + <Item> + <New class="org.eclipse.jetty.server.HttpConnectionFactory"> + <Arg name="config"><Ref refid="httpConfig" /></Arg> + </New> + </Item> + </Array> + </Arg> + <Set name="host"><Property name="jetty.host" /></Set> + <Set name="port"><Property name="jetty.port" default="8080" /></Set> + <Set name="idleTimeout"><Property name="http.timeout" default="30000"/></Set> + </New> + </Arg> + </Call> - <Call name="addConnector"> - <Arg> - <New class="org.eclipse.jetty.server.nio.SelectChannelConnector"> - <Set name="host"><Property name="jetty.host" /></Set> - <Set name="port"><Property name="jetty.port" default="8080"/></Set> - </New> - </Arg> - </Call> - <!-- =========================================================== --> <!-- Set handler Collection Structure --> <!-- =========================================================== --> @@ -36,25 +126,29 @@ <Set name="handlers"> <Array type="org.eclipse.jetty.server.Handler"> <Item> + <!-- This is the bigdata web application. --> <New id="WebAppContext" class="org.eclipse.jetty.webapp.WebAppContext"> - <!-- The location of the top-level of the bigdata webapp. --> - <Set name="resourceBase"> - <Property name="jetty.resourceBase" default="<%= node['systap-bigdataHA'][:fed_dir] %>/var/jetty" /> + <Set name="war"> + <!-- The location of the top-level of the bigdata webapp. --> + <Property name="jetty.resourceBase" default="<%= node['systap-bigdataHA'][:fed_dir] %>/var/jetty" /> </Set> - <Set name="contextPath">/</Set> + <Set name="contextPath">/bigdata</Set> <Set name="descriptor"><%= node['systap-bigdataHA'][:fed_dir] %>/var/jetty/WEB-INF/web.xml</Set> <Set name="parentLoaderPriority">true</Set> <Set name="extractWAR">false</Set> - <Set name="welcomeFiles"> - <Array type="java.lang.String"> - <Item>index.html</Item> - </Array> - </Set> </New> </Item> </Array> </Set> </New> </Set> - + + <!-- =========================================================== --> + <!-- extra server options --> + <!-- =========================================================== --> + <Set name="stopAtShutdown">true</Set> + <Set name="stopTimeout">5000</Set> + <Set name="dumpAfterStart"><Property name="jetty.dump.start" default="false"/></Set> + <Set name="dumpBeforeStop"><Property name="jetty.dump.stop" default="false"/></Set> + </Configure> Modified: branches/DEPLOYMENT_BRANCH_1_3_1/src/resources/etc/init.d/bigdataHA =================================================================== --- branches/DEPLOYMENT_BRANCH_1_3_1/src/resources/etc/init.d/bigdataHA 2014-05-06 13:54:41 UTC (rev 8207) +++ branches/DEPLOYMENT_BRANCH_1_3_1/src/resources/etc/init.d/bigdataHA 2014-05-06 17:57:47 UTC (rev 8208) @@ -95,7 +95,7 @@ fi fi if [ ! -f "$pidFile" ]; then - action $"`date` : `hostname` : bringing up services: " sudo -u $BD_USER -g $BD_GROUP bash -c "source /etc/default/bigdataHA ; $binDir/startHAServices" + action $"`date` : `hostname` : bringing up services: " sudo -u $BD_USER -g $BD_GROUP bash -c "source /etc/default/bigdataHA ; $binDir/startHAServices > /dev/null 2>&1 &" else echo $"`date` : `hostname` : running as $pid" fi This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |