Menu
▾
▴
bigdata-commit — Commit traffic.
This list is closed, nobody may subscribe to it.
| 2010 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
(139) |
Aug
(94) |
Sep
(232) |
Oct
(143) |
Nov
(138) |
Dec
(55) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2011 |
Jan
(127) |
Feb
(90) |
Mar
(101) |
Apr
(74) |
May
(148) |
Jun
(241) |
Jul
(169) |
Aug
(121) |
Sep
(157) |
Oct
(199) |
Nov
(281) |
Dec
(75) |
| 2012 |
Jan
(107) |
Feb
(122) |
Mar
(184) |
Apr
(73) |
May
(14) |
Jun
(49) |
Jul
(26) |
Aug
(103) |
Sep
(133) |
Oct
(61) |
Nov
(51) |
Dec
(55) |
| 2013 |
Jan
(59) |
Feb
(72) |
Mar
(99) |
Apr
(62) |
May
(92) |
Jun
(19) |
Jul
(31) |
Aug
(138) |
Sep
(47) |
Oct
(83) |
Nov
(95) |
Dec
(111) |
| 2014 |
Jan
(125) |
Feb
(60) |
Mar
(119) |
Apr
(136) |
May
(270) |
Jun
(83) |
Jul
(88) |
Aug
(30) |
Sep
(47) |
Oct
(27) |
Nov
(23) |
Dec
|
| 2015 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
(3) |
Oct
|
Nov
|
Dec
|
| 2016 |
Jan
|
Feb
|
Mar
(4) |
Apr
(1) |
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
|
From: <tho...@us...> - 2014-05-14 20:50:04
|
Revision: 8334
http://sourceforge.net/p/bigdata/code/8334
Author: thompsonbry
Date: 2014-05-14 20:50:02 +0000 (Wed, 14 May 2014)
Log Message:
-----------
javadoc
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 20:47:31 UTC (rev 8333)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 20:50:02 UTC (rev 8334)
@@ -112,8 +112,11 @@
* TODO In fact, we could automatically compute and use a reasonable
* value based on the quorum size as
* <code>ceil((1/replicationFactor)-.01)</code>. With this approach, the
- * local forward bias is automatic and has no more than a 1% deviation
- * from the optimal decision if a round-robin is NOT being applied.
+ * local forward bias is automatic. However, we still only want to do
+ * this if there is a round-robin over the services. Otherwise we will
+ * slam this host whenever its load gets below the threshold while not
+ * assigning any work to the other hosts until the next update of the
+ * {@link HostTable}.
*/
String LOCAL_FORWARD_THRESHOLD = "localForwardThreshold";
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 20:47:35
|
Revision: 8333
http://sourceforge.net/p/bigdata/code/8333
Author: thompsonbry
Date: 2014-05-14 20:47:31 +0000 (Wed, 14 May 2014)
Log Message:
-----------
javadoc and code cleanup related to AVAILABILITY vs LOAD
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/lbs/TestAbstractHostLBSPolicy.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 20:38:50 UTC (rev 8332)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 20:47:31 UTC (rev 8333)
@@ -111,8 +111,8 @@
*
* TODO In fact, we could automatically compute and use a reasonable
* value based on the quorum size as
- * <code>ceil((1/replicationFactor)-.05)</code>. With this approach, the
- * local forward bias is automatic and has no more than a 5% deviation
+ * <code>ceil((1/replicationFactor)-.01)</code>. With this approach, the
+ * local forward bias is automatic and has no more than a 1% deviation
* from the optimal decision if a round-robin is NOT being applied.
*/
String LOCAL_FORWARD_THRESHOLD = "localForwardThreshold";
@@ -670,7 +670,7 @@
if (log.isDebugEnabled())
log.debug("hostname=" + hostname + ", metrics="
- + metrics2[i] + ", score=" + hostScore.getScore());
+ + metrics2[i] + ", score=" + hostScore.getAvailability());
}
@@ -836,7 +836,7 @@
double sum = 0d;
for (HostScore tmp : hostScores) {
hostScore = tmp;
- sum += hostScore.getScore();
+ sum += hostScore.getAvailability();
if (sum >= d) {
// found desired host.
break;
@@ -948,7 +948,7 @@
: hostTable.thisHost;
if (thisHostScore != null
- && thisHostScore.getScore() >= localForwardThresholdRef.get()) {
+ && thisHostScore.getAvailability() >= localForwardThresholdRef.get()) {
servlet.forwardToLocalService(false/* isLeaderRequest */, request,
response);
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java 2014-05-14 20:38:50 UTC (rev 8332)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java 2014-05-14 20:47:31 UTC (rev 8333)
@@ -27,9 +27,11 @@
import com.bigdata.counters.AbstractStatisticsCollector;
/**
- * Helper class assigns a raw score (load) and a normalized score (normalized
- * load) to each host based on its per-host metrics and a rule for computing the
- * load of a host from those metrics.
+ * Helper class pairs a hostname and the normalized availabilty for that host.
+ * The availability is based on (normalized) <code>1 - LOAD</code> for the host.
+ * The <code>LOAD</code> is computed using the {@link IHostMetrics} and an
+ * {@link IHostScoringRule} for computing the workload of a host from those
+ * metrics. The availability is then computed from the LOAD and normalized.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
*/
@@ -43,63 +45,19 @@
*/
private final boolean thisHost;
-// /**
-// * The raw score for some host. This is a measure of the load on a host. The
-// * measure is computed based on {@link #metrics} using the
-// * {@link #scoringRule}.
-// */
-// private final double rawScore;
-
/**
- * The normalized score for that host.
- * <p>
- * Note: The {@link #rawScore} is a measure of the <em>load</em> on a host.
- * The normalized {@link #score} is a measure of the normalized
- * <em>load</em> on a host. Load balancing decision are based on this
- * normalized {@link #score} (work is assigned to a host in inverse
- * proportion to its normalized {@link #score}).
+ * The normalized availability for the host.
*/
- private final double score;
+ private final double availability;
-// /**
-// * The {@link IHostScoringRule} used to convert the {@link #metrics} into
-// * the {@link #rawScore}.
-// */
-// private final IHostScoringRule scoringRule;
-//
-// /**
-// * The {@link IHostMetrics} associated with this host (this is
-// * informational. The {@link #metrics} provide the detailed per-host
-// * performance metrics that were intepreted by the {@link #scoringRule} .
-// */
-// final private IHostMetrics metrics;
-
-// /** The rank in [0:#scored]. This is an index into the Scores[]. */
-// public int rank = -1;
-//
-// /** The normalized double precision rank in [0.0:1.0]. */
-// public double drank = -1d;
-
-// /**
-// * Return the raw score (aka load) for a host. This raw score is an
-// * unnormalized measure of the load on that host. The measure is computed
-// * based on {@link IHostMetrics} using some {@link IHostScoringRule}.
-// */
-// public double getRawScore() {
-// return rawScore;
-// }
-
/**
- * Return the normalized load for the host.
- * <p>
- * Note: The {@link #getRawScore() rawScore} is a measure of the
- * <em>load</em> on a host. The normalized {@link #getScore() score} is the
- * normalized load for a host. Load balancing decision are based on this
- * normalized {@link #getScore() score} (work is assigned to hosts in
- * inverse proportion to the normalized load of the host).
+ * Return the normalized availability for the host. Load balancing decision
+ * are based on this normalized {@link #getAvailability() score} (work is
+ * assigned to hosts in inverse proportion to the normalized load of the
+ * host).
*/
- public double getScore() {
- return score;
+ public double getAvailability() {
+ return availability;
}
/** Return the hostname. */
@@ -107,23 +65,6 @@
return hostname;
}
-// /**
-// * The {@link IHostMetrics} associated with this host (optional). The
-// * {@link #getMetrics()} provide the detailed per-host performance metrics
-// * that were intepreted by the {@link #getScoringRule()} .
-// */
-// public IHostMetrics getMetrics() {
-// return metrics;
-// }
-//
-// /**
-// * The {@link IHostScoringRule} used to convert the {@link #getMetrics()}
-// * into the {@link #getRawScore()} (optional).
-// */
-// public IHostScoringRule getScoringRule() {
-// return scoringRule;
-// }
-
/**
* Return <code>true</code> iff the host is this host.
*/
@@ -137,12 +78,7 @@
return "HostScore"//
+ "{hostname=" + hostname //
+ ", thisHost=" + thisHost//
-// + ", rawScore=" + rawScore //
- + ", score=" + score //
-// + ", rank=" + rank //
-// + ", drank=" + drank //
-// + ", metrics=" + metrics //
-// + ", scoringRule=" + scoringRule //
+ + ", availabilty=" + availability //
+ "}";
}
@@ -151,25 +87,12 @@
*
* @param hostname
* The hostname (required, must be non-empty).
- * @param score The normalized availability score for this
- * host.
-// * @param rawScore
-// * The unnormalized load for that host.
-// * @param totalRawScore
-// * The total unnormalized load across all hosts.
+ * @param availability
+ * The normalized availability score for this host.
*/
-// * @param metrics
-// * The performance metrics used to compute the unnormalized load
-// * for each host (optional).
-// * @param scoringRule
-// * The rule used to score those metrics (optional).
public HostScore(//
final String hostname,//
- final double score
-// final double rawScore,//
-// final double totalRawScore //
-// final IHostMetrics metrics,//
-// final IHostScoringRule scoringRule//
+ final double availability
) {
if (hostname == null)
@@ -178,28 +101,20 @@
if (hostname.trim().length() == 0)
throw new IllegalArgumentException();
- if (score < 0d || score > 1d)
+ if (availability < 0d || availability > 1d)
throw new IllegalArgumentException();
this.hostname = hostname;
- this.score = score;
+ this.availability = availability;
-// this.rawScore = rawScore;
-
this.thisHost = AbstractStatisticsCollector.fullyQualifiedHostName
.equals(hostname);
-
-// this.scoringRule = scoringRule;
-//
-// this.metrics = metrics;
-// score = normalize(rawScore, totalRawScore);
-
}
/**
- * Places elements into order by decreasing {@link #getScore() normalized
+ * Places elements into order by decreasing {@link #getAvailability() normalized
* load}. The {@link #getHostname()} is used to break any ties.
*/
public final static Comparator<HostScore> COMPARE_BY_SCORE = new Comparator<HostScore>() {
@@ -207,11 +122,11 @@
@Override
public int compare(final HostScore t1, final HostScore t2) {
- if (t1.score < t2.score) {
+ if (t1.availability < t2.availability) {
return 1;
- } else if (t1.score > t2.score) {
+ } else if (t1.availability > t2.availability) {
return -1;
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/lbs/TestAbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/lbs/TestAbstractHostLBSPolicy.java 2014-05-14 20:38:50 UTC (rev 8332)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/lbs/TestAbstractHostLBSPolicy.java 2014-05-14 20:47:31 UTC (rev 8333)
@@ -77,7 +77,7 @@
final HostScore hostScore1 = new HostScore("H1", 1d);
// should sum to 1.0
- assertEquals(1d, hostScore1.getScore());
+ assertEquals(1d, hostScore1.getAvailability());
final HostScore[] hostScores = new HostScore[] { //
hostScore1 //
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 20:38:54
|
Revision: 8332
http://sourceforge.net/p/bigdata/code/8332
Author: thompsonbry
Date: 2014-05-14 20:38:50 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Javadoc. Fix to COMPARATOR names.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 20:33:20 UTC (rev 8331)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 20:38:50 UTC (rev 8332)
@@ -108,6 +108,12 @@
* their normalized availability scores will be <code>.3333</code>. If
* the score for a given host is close to this normalized availability,
* then a local forward is a reasonable choice.
+ *
+ * TODO In fact, we could automatically compute and use a reasonable
+ * value based on the quorum size as
+ * <code>ceil((1/replicationFactor)-.05)</code>. With this approach, the
+ * local forward bias is automatic and has no more than a 5% deviation
+ * from the optimal decision if a round-robin is NOT being applied.
*/
String LOCAL_FORWARD_THRESHOLD = "localForwardThreshold";
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java 2014-05-14 20:33:20 UTC (rev 8331)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java 2014-05-14 20:38:50 UTC (rev 8332)
@@ -202,7 +202,7 @@
* Places elements into order by decreasing {@link #getScore() normalized
* load}. The {@link #getHostname()} is used to break any ties.
*/
- public final static Comparator<HostScore> COMPARE_BY_HOSTNAME = new Comparator<HostScore>() {
+ public final static Comparator<HostScore> COMPARE_BY_SCORE = new Comparator<HostScore>() {
@Override
public int compare(final HostScore t1, final HostScore t2) {
@@ -227,7 +227,7 @@
* Orders by hostname. This provides a stable way of viewing the data in
* <code>/bigdata/status</code>.
*/
- public final Comparator<HostScore> COMPARE_BY_SCORE = new Comparator<HostScore>() {
+ public final static Comparator<HostScore> COMPARE_BY_HOSTNAME = new Comparator<HostScore>() {
@Override
public int compare(final HostScore t1, final HostScore t2) {
@@ -235,7 +235,7 @@
return t1.hostname.compareTo(t2.hostname);
}
-
+
};
}
\ No newline at end of file
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 20:33:24
|
Revision: 8331
http://sourceforge.net/p/bigdata/code/8331
Author: thompsonbry
Date: 2014-05-14 20:33:20 +0000 (Wed, 14 May 2014)
Log Message:
-----------
javadoc
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 20:27:18 UTC (rev 8330)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 20:33:20 UTC (rev 8331)
@@ -95,15 +95,19 @@
* when considering the normalized workload of the hosts. The value must
* be in [0:1] and represents a normalized availability threshold for
* the hosts having services that are joined with the met quorum. This
- * may be set to ONT (1.0) to disable this bias. The default is
- * {@value #DEFAULT_LOCAL_FORWARD_THRESHOLD}. A reasonable value might
- * be on the order of <code>.7</code> or <code>.8</code>.
+ * may be set to ONE (1.0) to disable this bias. The default is
+ * {@value #DEFAULT_LOCAL_FORWARD_THRESHOLD}.
* <p>
* This bias is designed for use when an external round-robin policy is
* distributing the requests evenly across the services. In this case,
* the round-robin smooths out most of the workload and the
* {@link IHALoadBalancerPolicy} takes over only when there is a severe
* workload imbalance (as defined by the value of this parameter).
+ * <p>
+ * For example, if you have 3 hosts and they are equally available, then
+ * their normalized availability scores will be <code>.3333</code>. If
+ * the score for a given host is close to this normalized availability,
+ * then a local forward is a reasonable choice.
*/
String LOCAL_FORWARD_THRESHOLD = "localForwardThreshold";
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 20:27:21
|
Revision: 8330
http://sourceforge.net/p/bigdata/code/8330
Author: thompsonbry
Date: 2014-05-14 20:27:18 +0000 (Wed, 14 May 2014)
Log Message:
-----------
More games with the HA LBS init-param overrides.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-05-14 20:19:50 UTC (rev 8329)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-05-14 20:27:18 UTC (rev 8330)
@@ -667,16 +667,21 @@
* the environment variables first for a fully qualified value for the
* parameter using <i>owningClass</i>.<i>name</i> (or just <i>name</i> if
* <i>owningClass</i> is <code>null</code>). If no value is found for that
- * variable, it checks the {@link ServletContext} for <i>name</i>. If no
- * value is found again, it returns the default value specified by the
- * caller. This makes it possible to configure the behavior of the
- * {@link HALoadBalancerServlet} using environment variables.
+ * variable, it checks the {@link ServletContext} for <i>name</i> (if the
+ * owningClass:=HALoadBalancerServlet) and otherwise for
+ * <i>owningClass.name</i>. If no value is found again, it returns the
+ * default value specified by the caller. This makes it possible to
+ * configure the behavior of the {@link HALoadBalancerServlet} using
+ * environment variables.
*
* @param servletConfig
* The {@link ServletConfig}.
* @param owningClass
* The class that declares the init-param (required). This serves
- * as a namespace when searching the environment variables.
+ * as a namespace when searching the environment variables. This
+ * is also used to impose a namespace when searching
+ * <code>web.xml</code> when
+ * <code>owningClass!=HALoadBalancerServlet</code>.
* @param name
* The name of the servlet <code>init-param</code>.
* @param def
@@ -692,8 +697,26 @@
if (s == null || s.trim().length() == 0) {
// Look at ServletConfig for the configured value.
- s = servletConfig.getInitParameter(name);
+ if (owningClass == HALoadBalancerServlet.class) {
+ /*
+ * The HALoadBalancerServlet does not use a namespace prefix for
+ * its web.xml declarations.
+ */
+ s = servletConfig.getInitParameter(name);
+
+ } else {
+
+ /*
+ * The other policy objects DO use their owningClass as a
+ * namespace prefix. This is done to avoid collisions among the
+ * different policy classes.
+ */
+ s = servletConfig.getInitParameter(owningClass.getName() + "."
+ + name);
+
+ }
+
}
if (s == null || s.trim().length() == 0) {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 20:19:50 UTC (rev 8329)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 20:27:18 UTC (rev 8330)
@@ -65,6 +65,12 @@
*/
private static final long serialVersionUID = 1L;
+ /**
+ *
+ * @see HALoadBalancerServlet#getConfigParam(ServletConfig, Class, String,
+ * String) for how these <code>init-param</code> values can be set in
+ * <code>web.xml</code> and via environment variables.
+ */
public interface InitParams extends AbstractLBSPolicy.InitParams {
/**
@@ -81,8 +87,7 @@
* Note: The default policy is specific to the concrete instance of the
* outer class.
*/
- String HOST_SCORING_RULE = AbstractHostLBSPolicy.class.getName()
- + ".hostScoringRule";
+ String HOST_SCORING_RULE = "hostScoringRule";
/**
* Read requests are forwarded to the local service if the availability
@@ -100,8 +105,7 @@
* {@link IHALoadBalancerPolicy} takes over only when there is a severe
* workload imbalance (as defined by the value of this parameter).
*/
- String LOCAL_FORWARD_THRESHOLD = AbstractHostLBSPolicy.class.getName()
- + ".localForwardThreshold";
+ String LOCAL_FORWARD_THRESHOLD = "localForwardThreshold";
String DEFAULT_LOCAL_FORWARD_THRESHOLD = "1.0";
@@ -111,8 +115,7 @@
* the joined services (default
* {@value #DEFAULT_HOST_DISCOVERY_INITIAL_DELAY}).
*/
- String HOST_DISCOVERY_INITIAL_DELAY = AbstractHostLBSPolicy.class.getName()
- + ".hostDiscoveryInitialDelay";
+ String HOST_DISCOVERY_INITIAL_DELAY = "hostDiscoveryInitialDelay";
String DEFAULT_HOST_DISCOVERY_INITIAL_DELAY = "10000"; // ms.
@@ -121,8 +124,7 @@
* in-memory snapshots of the performance metrics for the joined
* services (default {@value #DEFAULT_HOST_DISCOVERY_DELAY}).
*/
- String HOST_DISCOVERY_DELAY = AbstractHostLBSPolicy.class.getName()
- + ".hostDiscoveryDelay";
+ String HOST_DISCOVERY_DELAY = "hostDiscoveryDelay";
String DEFAULT_HOST_DISCOVERY_DELAY = "10000"; // ms.
@@ -236,6 +238,8 @@
sb.append(",localForwardThreshold=" + localForwardThresholdRef.get());
+ sb.append(",hostDiscoveryInitialDelay=" + hostDiscoveryInitialDelay);
+
sb.append(",hostDiscoveryDelay=" + hostDiscoveryDelay);
sb.append(",scoringRule=" + scoringRuleRef.get());
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 20:19:53
|
Revision: 8329
http://sourceforge.net/p/bigdata/code/8329
Author: thompsonbry
Date: 2014-05-14 20:19:50 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Bug fix to override logic for HA LBS.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-05-14 20:17:01 UTC (rev 8328)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-05-14 20:19:50 UTC (rev 8329)
@@ -686,10 +686,8 @@
public static String getConfigParam(final ServletConfig servletConfig,
final Class<?> owningClass, final String name, final String def) {
- String s = null;
-
// Look at environment variables for an override.
- System.getProperty(owningClass.getName() + "." + name);
+ String s = System.getProperty(owningClass.getName() + "." + name);
if (s == null || s.trim().length() == 0) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 20:17:05
|
Revision: 8328
http://sourceforge.net/p/bigdata/code/8328
Author: thompsonbry
Date: 2014-05-14 20:17:01 +0000 (Wed, 14 May 2014)
Log Message:
-----------
javadoc
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-05-14 20:16:21 UTC (rev 8327)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-05-14 20:17:01 UTC (rev 8328)
@@ -675,8 +675,8 @@
* @param servletConfig
* The {@link ServletConfig}.
* @param owningClass
- * The class that declares the init-param. This serves as
- * a namespace when searching the environment variables.
+ * The class that declares the init-param (required). This serves
+ * as a namespace when searching the environment variables.
* @param name
* The name of the servlet <code>init-param</code>.
* @param def
@@ -689,18 +689,8 @@
String s = null;
// Look at environment variables for an override.
- if (owningClass != null) {
+ System.getProperty(owningClass.getName() + "." + name);
- // namespace is the owningClass.
- System.getProperty(owningClass.getName() + "." + name);
-
- } else {
-
- // no namespace.
- System.getProperty(name);
-
- }
-
if (s == null || s.trim().length() == 0) {
// Look at ServletConfig for the configured value.
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 20:16:24
|
Revision: 8327
http://sourceforge.net/p/bigdata/code/8327
Author: thompsonbry
Date: 2014-05-14 20:16:21 +0000 (Wed, 14 May 2014)
Log Message:
-----------
and corrected to use the HALoadBalancerServlet class as a namespace prefix when searching the environment variables for an override.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-05-14 20:10:34 UTC (rev 8326)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-05-14 20:16:21 UTC (rev 8327)
@@ -418,8 +418,10 @@
{
// Get the as-configured policy.
- final IHALoadBalancerPolicy policy = newInstance(servletConfig,
- null/* owningClass */, IHALoadBalancerPolicy.class,
+ final IHALoadBalancerPolicy policy = newInstance(//
+ servletConfig, //
+ HALoadBalancerServlet.class,// owningClass
+ IHALoadBalancerPolicy.class,//
InitParams.POLICY, InitParams.DEFAULT_POLICY);
// Set the as-configured policy.
@@ -428,8 +430,10 @@
}
{
- final IHARequestURIRewriter rewriter = newInstance(servletConfig,
- null/* owningClass */, IHARequestURIRewriter.class,
+ final IHARequestURIRewriter rewriter = newInstance(//
+ servletConfig,//
+ HALoadBalancerServlet.class, // owningClass
+ IHARequestURIRewriter.class,//
InitParams.REWRITER, InitParams.DEFAULT_REWRITER);
setRewriter(rewriter);
@@ -671,9 +675,8 @@
* @param servletConfig
* The {@link ServletConfig}.
* @param owningClass
- * The name of the class that provides the namespace for the
- * <code>init-param</code> (optional - not used by the outer
- * servlet, just by the inner policy classes).
+ * The class that declares the init-param. This serves as
+ * a namespace when searching the environment variables.
* @param name
* The name of the servlet <code>init-param</code>.
* @param def
@@ -681,8 +684,7 @@
* @return
*/
public static String getConfigParam(final ServletConfig servletConfig,
- final Class<? extends IHAPolicyLifeCycle> owningClass,
- final String name, final String def) {
+ final Class<?> owningClass, final String name, final String def) {
String s = null;
@@ -725,6 +727,9 @@
*
* @param servletConfig
* The {@link ServletConfig}.
+ * @param owningClass
+ * The class that declares the init-param. This serves as
+ * a namespace when searching the environment variables.
* @param iface
* The interface that the type must implement.
* @param name
@@ -739,7 +744,7 @@
*/
@SuppressWarnings("unchecked")
public static <T> T newInstance(final ServletConfig servletConfig,
- final Class<? extends IHAPolicyLifeCycle> owningClass,
+ final Class<?> owningClass,
final Class<? extends T> iface, final String name, final String def)
throws ServletException {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 20:10:37
|
Revision: 8326
http://sourceforge.net/p/bigdata/code/8326
Author: thompsonbry
Date: 2014-05-14 20:10:34 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Modified to use the namespace for the servlet init-params only for the policy classes.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-05-14 19:52:16 UTC (rev 8325)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-05-14 20:10:34 UTC (rev 8326)
@@ -419,8 +419,8 @@
{
// Get the as-configured policy.
final IHALoadBalancerPolicy policy = newInstance(servletConfig,
- IHALoadBalancerPolicy.class, InitParams.POLICY,
- InitParams.DEFAULT_POLICY);
+ null/* owningClass */, IHALoadBalancerPolicy.class,
+ InitParams.POLICY, InitParams.DEFAULT_POLICY);
// Set the as-configured policy.
setLBSPolicy(policy);
@@ -429,8 +429,8 @@
{
final IHARequestURIRewriter rewriter = newInstance(servletConfig,
- IHARequestURIRewriter.class, InitParams.REWRITER,
- InitParams.DEFAULT_REWRITER);
+ null/* owningClass */, IHARequestURIRewriter.class,
+ InitParams.REWRITER, InitParams.DEFAULT_REWRITER);
setRewriter(rewriter);
@@ -661,30 +661,44 @@
/**
* Return the configured value of the named parameter. This method checks
* the environment variables first for a fully qualified value for the
- * parameter using <code>HALoadBalancerServer</code><i>name</i>. If no value
- * is found for that variable, it checks the {@link ServletContext} for
- * <i>name</i>. If no value is found again, it returns the default value
- * specified by the caller. This makes it possible to configure the behavior
- * of the {@link HALoadBalancerServlet} using environment variables.
+ * parameter using <i>owningClass</i>.<i>name</i> (or just <i>name</i> if
+ * <i>owningClass</i> is <code>null</code>). If no value is found for that
+ * variable, it checks the {@link ServletContext} for <i>name</i>. If no
+ * value is found again, it returns the default value specified by the
+ * caller. This makes it possible to configure the behavior of the
+ * {@link HALoadBalancerServlet} using environment variables.
*
* @param servletConfig
* The {@link ServletConfig}.
- *
- * @param iface
- * The interface that the type must implement.
+ * @param owningClass
+ * The name of the class that provides the namespace for the
+ * <code>init-param</code> (optional - not used by the outer
+ * servlet, just by the inner policy classes).
* @param name
- * The name of the servlet init parameter.
+ * The name of the servlet <code>init-param</code>.
* @param def
- * The default value for the servlet init parameter.
+ * The default value for the servlet <code>init-param</code>.
* @return
*/
public static String getConfigParam(final ServletConfig servletConfig,
+ final Class<? extends IHAPolicyLifeCycle> owningClass,
final String name, final String def) {
+ String s = null;
+
// Look at environment variables for an override.
- String s = System.getProperty(HALoadBalancerServlet.class.getName()
- + "." + name);
+ if (owningClass != null) {
+ // namespace is the owningClass.
+ System.getProperty(owningClass.getName() + "." + name);
+
+ } else {
+
+ // no namespace.
+ System.getProperty(name);
+
+ }
+
if (s == null || s.trim().length() == 0) {
// Look at ServletConfig for the configured value.
@@ -725,10 +739,11 @@
*/
@SuppressWarnings("unchecked")
public static <T> T newInstance(final ServletConfig servletConfig,
+ final Class<? extends IHAPolicyLifeCycle> owningClass,
final Class<? extends T> iface, final String name, final String def)
throws ServletException {
- final String s = getConfigParam(servletConfig, name, def);
+ final String s = getConfigParam(servletConfig, owningClass, name, def);
final T t;
final Class<? extends T> cls;
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 19:52:16 UTC (rev 8325)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 20:10:34 UTC (rev 8326)
@@ -290,7 +290,9 @@
{
final String s = HALoadBalancerServlet.getConfigParam(
- servletConfig, InitParams.LOCAL_FORWARD_THRESHOLD,
+ servletConfig, //
+ AbstractHostLBSPolicy.class,// owningClass
+ InitParams.LOCAL_FORWARD_THRESHOLD,
InitParams.DEFAULT_LOCAL_FORWARD_THRESHOLD);
final double d = Double.valueOf(s);
@@ -304,7 +306,9 @@
{
- scoringRuleRef.set(HALoadBalancerServlet.newInstance(servletConfig,
+ scoringRuleRef.set(HALoadBalancerServlet.newInstance(//
+ servletConfig,//
+ AbstractHostLBSPolicy.class,// owningClass
IHostScoringRule.class, InitParams.HOST_SCORING_RULE,
getDefaultScoringRule()));
@@ -317,7 +321,9 @@
{
final String s = HALoadBalancerServlet.getConfigParam(
- servletConfig, InitParams.HOST_DISCOVERY_INITIAL_DELAY,
+ servletConfig, //
+ AbstractHostLBSPolicy.class,// owningClass
+ InitParams.HOST_DISCOVERY_INITIAL_DELAY,
InitParams.DEFAULT_HOST_DISCOVERY_INITIAL_DELAY);
hostDiscoveryInitialDelay = Long.valueOf(s);
@@ -330,8 +336,10 @@
{
- final String s = HALoadBalancerServlet.getConfigParam(
- servletConfig, InitParams.HOST_DISCOVERY_DELAY,
+ final String s = HALoadBalancerServlet.getConfigParam(//
+ servletConfig, //
+ AbstractHostLBSPolicy.class,// owningClass
+ InitParams.HOST_DISCOVERY_DELAY,//
InitParams.DEFAULT_HOST_DISCOVERY_DELAY);
hostDiscoveryDelay = Long.valueOf(s);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 19:52:19
|
Revision: 8325
http://sourceforge.net/p/bigdata/code/8325
Author: thompsonbry
Date: 2014-05-14 19:52:16 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Modified the localForwardThreshold to be stated and interpreted in terms of availabilty rather than load.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 19:39:53 UTC (rev 8324)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 19:52:16 UTC (rev 8325)
@@ -85,25 +85,25 @@
+ ".hostScoringRule";
/**
- * Read requests are forwarded to the local service if the load on that
- * service is less than the configured threshold when considering the
- * normalized workload of the hosts. The value must be in (0:1) and
- * represents a normalized workload threshold for the hosts having
- * services that are joined with the met quorum. This may be set to ZERO
- * (0) to disable this bias. The default is
- * {@value #DEFAULT_LOCAL_FORWARD_THRESHOLD}.
+ * Read requests are forwarded to the local service if the availability
+ * on that service is greater than or equal to the configured threshold
+ * when considering the normalized workload of the hosts. The value must
+ * be in [0:1] and represents a normalized availability threshold for
+ * the hosts having services that are joined with the met quorum. This
+ * may be set to ONT (1.0) to disable this bias. The default is
+ * {@value #DEFAULT_LOCAL_FORWARD_THRESHOLD}. A reasonable value might
+ * be on the order of <code>.7</code> or <code>.8</code>.
* <p>
* This bias is designed for use when an external round-robin policy is
* distributing the requests evenly across the services. In this case,
- * the round-robin smooths out most of the load and the
- * {@link HALoadBalancerServlet} {@link #POLICY} takes over only when
- * there is a severe load imbalance (as defined by the value of this
- * parameter).
+ * the round-robin smooths out most of the workload and the
+ * {@link IHALoadBalancerPolicy} takes over only when there is a severe
+ * workload imbalance (as defined by the value of this parameter).
*/
String LOCAL_FORWARD_THRESHOLD = AbstractHostLBSPolicy.class.getName()
+ ".localForwardThreshold";
- String DEFAULT_LOCAL_FORWARD_THRESHOLD = "0";
+ String DEFAULT_LOCAL_FORWARD_THRESHOLD = "1.0";
/**
* The initial delay in milliseconds before the first scheduled task
@@ -907,9 +907,11 @@
/**
* {@inheritDoc}
* <p>
- * If the normalized workload for this host is under a configured threshold,
- * then we forward the request to this service. This help to reduce the
- * latency of the request since it is not being proxied.
+ * If the normalized availability for this host is over a configured
+ * threshold, then we forward the request to the local service. This help to
+ * reduce the latency of the request since it is not being proxied.
+ *
+ * @see InitParams#LOCAL_FORWARD_THRESHOLD
*/
@Override
protected boolean conditionallyForwardReadRequest(
@@ -924,7 +926,7 @@
: hostTable.thisHost;
if (thisHostScore != null
- && thisHostScore.getScore() <= localForwardThresholdRef.get()) {
+ && thisHostScore.getScore() >= localForwardThresholdRef.get()) {
servlet.forwardToLocalService(false/* isLeaderRequest */, request,
response);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 19:39:56
|
Revision: 8324
http://sourceforge.net/p/bigdata/code/8324
Author: thompsonbry
Date: 2014-05-14 19:39:53 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Passing through a bunch of environment variables that should allow us to control the HA LBS configuration without edits to web.xml. See #624 (HA LBS)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices 2014-05-14 19:23:34 UTC (rev 8323)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices 2014-05-14 19:39:53 UTC (rev 8324)
@@ -81,6 +81,12 @@
-DGANGLIA_LISTEN=${GANGLIA_LISTEN}\
-DSYSSTAT_DIR=${SYSSTAT_DIR}\
-Dcom.bigdata.counters.linux.sysstat.path=${SYSSTAT_DIR}\
+ -Dcom.bigdata.rdf.sail.webapp.HALoadBalancerServlet.policy=${LBS_POLICY}\
+ -Dcom.bigdata.rdf.sail.webapp.HALoadBalancerServlet.rewriter=${LBS_REWRITER}\
+ -Dcom.bigdata.rdf.sail.webapp.lbs.AbstractHostLBSPolicy.hostScoringRule=${LBS_HOST_SCORING_RULE}\
+ -Dcom.bigdata.rdf.sail.webapp.lbs.AbstractHostLBSPolicy.localForwardThreshold=${LBS_LOCAL_FORWARD_THRESHOLD}\
+ -Dcom.bigdata.rdf.sail.webapp.lbs.AbstractHostLBSPolicy.hostDiscoveryInitialDelay=${LBS_HOST_DISCOVERY_INITIAL_DELAY}\
+ -Dcom.bigdata.rdf.sail.webapp.lbs.AbstractHostLBSPolicy.hostDiscoveryDelay=${LBS_HOST_DISCOVERY_DELAY}\
"
##
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 19:23:39
|
Revision: 8323
http://sourceforge.net/p/bigdata/code/8323
Author: thompsonbry
Date: 2014-05-14 19:23:34 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Defined comparators for HostScore based on the score and based on the hostname. The former is stable in terms of high to low availability. The latter is stable in terms of the order of the hosts.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 19:22:27 UTC (rev 8322)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 19:23:34 UTC (rev 8323)
@@ -23,6 +23,7 @@
package com.bigdata.rdf.sail.webapp.lbs;
import java.io.IOException;
+import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -113,7 +114,7 @@
String HOST_DISCOVERY_INITIAL_DELAY = AbstractHostLBSPolicy.class.getName()
+ ".hostDiscoveryInitialDelay";
- String DEFAULT_HOST_DISCOVERY_INITIAL_DELAY = "60000"; // ms.
+ String DEFAULT_HOST_DISCOVERY_INITIAL_DELAY = "10000"; // ms.
/**
* The delay in milliseconds between scheduled tasks that update the
@@ -123,7 +124,7 @@
String HOST_DISCOVERY_DELAY = AbstractHostLBSPolicy.class.getName()
+ ".hostDiscoveryDelay";
- String DEFAULT_HOST_DISCOVERY_DELAY = "5000"; // ms.
+ String DEFAULT_HOST_DISCOVERY_DELAY = "10000"; // ms.
}
@@ -679,6 +680,14 @@
//
// }
+ /*
+ * Sort into order by hostname (useful for /bigdata/status view).
+ *
+ * Note: The ordering is not really material to anything. Stochastic
+ * load balancing decisions are made without regard to this ordering.
+ */
+ Arrays.sort(scores, HostScore.COMPARE_BY_HOSTNAME);
+
return new HostTable(thisHostScore, scores);
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java 2014-05-14 19:22:27 UTC (rev 8322)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java 2014-05-14 19:23:34 UTC (rev 8323)
@@ -22,6 +22,8 @@
*/
package com.bigdata.rdf.sail.webapp.lbs;
+import java.util.Comparator;
+
import com.bigdata.counters.AbstractStatisticsCollector;
/**
@@ -31,7 +33,7 @@
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
*/
-public class HostScore implements Comparable<HostScore> {
+public class HostScore {
/** The hostname. */
private final String hostname;
@@ -196,63 +198,44 @@
}
-// /**
-// * Computes the normalized {@link #score} from the {@link #rawScore} in the
-// * context of total over the {@link #rawScore}s for some set of hosts.
-// *
-// * @param rawScore
-// * The raw score.
-// * @param totalRawScore
-// * The raw score computed from the totals.
-// *
-// * @return The normalized score.
-// */
-// static private double normalize(final double rawScore,
-// final double totalRawScore) {
-//
-// if (totalRawScore == 0d) {
-//
-// return 0d;
-//
-// }
-//
-// final double score = rawScore / totalRawScore;
-//
-// if (score < 0 || score > 1) {
-//
-// throw new RuntimeException("score(" + score + ") := rawScore("
-// + rawScore + ") / totalRawScore(" + totalRawScore + ")");
-//
-// }
-//
-// return score;
-//
-// }
-
/**
* Places elements into order by decreasing {@link #getScore() normalized
- * load}. The {@link #getHostname()} is used to break any ties (but this
- * does not help when all services are on the same host).
- * <p>
- * Note: The ordering is not really material to anything. Stochastic load
- * balancing decisions can be made without regard to this ordering using the
- * {@link #getScore() normalized load}.
+ * load}. The {@link #getHostname()} is used to break any ties.
*/
- @Override
- public int compareTo(final HostScore arg0) {
+ public final static Comparator<HostScore> COMPARE_BY_HOSTNAME = new Comparator<HostScore>() {
- if (score < arg0.score) {
+ @Override
+ public int compare(final HostScore t1, final HostScore t2) {
- return 1;
+ if (t1.score < t2.score) {
- } else if (score > arg0.score) {
+ return 1;
- return -1;
+ } else if (t1.score > t2.score) {
+ return -1;
+
+ }
+
+ return t1.hostname.compareTo(t2.hostname);
+
}
- return hostname.compareTo(arg0.hostname);
+ };
- }
+ /**
+ * Orders by hostname. This provides a stable way of viewing the data in
+ * <code>/bigdata/status</code>.
+ */
+ public final Comparator<HostScore> COMPARE_BY_SCORE = new Comparator<HostScore>() {
+ @Override
+ public int compare(final HostScore t1, final HostScore t2) {
+
+ return t1.hostname.compareTo(t2.hostname);
+
+ }
+
+ };
+
}
\ No newline at end of file
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 19:22:29
|
Revision: 8322
http://sourceforge.net/p/bigdata/code/8322
Author: thompsonbry
Date: 2014-05-14 19:22:27 +0000 (Wed, 14 May 2014)
Log Message:
-----------
removed empty comment blocks.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/jetty.xml
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/jetty.xml
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/jetty.xml 2014-05-14 18:52:23 UTC (rev 8321)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/jetty.xml 2014-05-14 19:22:27 UTC (rev 8322)
@@ -21,9 +21,7 @@
<!-- Consult the javadoc of o.e.j.util.thread.QueuedThreadPool -->
<!-- for all configuration that may be set here. -->
<!-- =========================================================== -->
- <!-- uncomment to change type of threadpool -->
<Arg name="threadpool"><New id="threadpool" class="org.eclipse.jetty.util.thread.QueuedThreadPool"/></Arg>
- <!-- -->
<Get name="ThreadPool">
<Set name="minThreads" type="int"><SystemProperty name="jetty.threads.min" default="10"/></Set>
<Set name="maxThreads" type="int"><SystemProperty name="jetty.threads.max" default="64"/></Set>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 18:52:26
|
Revision: 8321
http://sourceforge.net/p/bigdata/code/8321
Author: thompsonbry
Date: 2014-05-14 18:52:23 +0000 (Wed, 14 May 2014)
Log Message:
-----------
relocating a debug message.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 18:51:05 UTC (rev 8320)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 18:52:23 UTC (rev 8321)
@@ -566,9 +566,6 @@
final IHostMetrics metrics = e.getValue();
- if (log.isDebugEnabled())
- log.debug("hostname=" + hostname + ", metrics=" + metrics);
-
// flag host if no load information is available.
double hostScore = metrics == null ? NO_INFO : scoringRule
.getScore(metrics);
@@ -648,6 +645,10 @@
}
+ if (log.isDebugEnabled())
+ log.debug("hostname=" + hostname + ", metrics="
+ + metrics2[i] + ", score=" + hostScore.getScore());
+
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 18:51:10
|
Revision: 8320
http://sourceforge.net/p/bigdata/code/8320
Author: thompsonbry
Date: 2014-05-14 18:51:05 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Bug fix to the CountersLBSPolicy. It was failing to look inside of the hostname prefix path for the counter sets and thus always failing to find the counters and assigning the same load to all hosts.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/policy/counters/CountersLBSPolicy.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/policy/counters/CountersLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/policy/counters/CountersLBSPolicy.java 2014-05-14 17:09:02 UTC (rev 8319)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/policy/counters/CountersLBSPolicy.java 2014-05-14 18:51:05 UTC (rev 8320)
@@ -48,6 +48,7 @@
import com.bigdata.bop.fed.QueryEngineFactory;
import com.bigdata.counters.CounterSet;
import com.bigdata.counters.DefaultInstrumentFactory;
+import com.bigdata.counters.ICounterNode;
import com.bigdata.journal.IIndexManager;
import com.bigdata.journal.Journal;
import com.bigdata.journal.PlatformStatsPlugIn;
@@ -102,7 +103,13 @@
*
* TODO Does not track per-service metrics unless we change to the service
* {@link UUID} as the key. This means that we can not monitor the GC load
- * associated with a specific JVM instance.
+ * associated with a specific JVM instance. [Another problem is that the
+ * metrics that we are collecting have the hostname as a prefix. The service
+ * metrics do not. This will cause problems in the path prefix in the
+ * {@link CounterSet} when we try to resolve the performance counter name.
+ * We could work around that by using a regex pattern to match the counters
+ * of interest, ignoring where they appear in the {@link CounterSet}
+ * hierarchy.]
*/
private final ConcurrentHashMap<String/* hostname */, IHostMetrics> hostMetricsMap = new ConcurrentHashMap<String, IHostMetrics>();
@@ -179,10 +186,25 @@
log.warn("No data: hostname=" + hostname);
continue;
}
-
- // Add to the map.
+
+ // Look for the child named by the hostname.
+ final ICounterNode childNode = counterSet.getPath(hostname);
+ if (childNode == null) {
+ log.warn("No data: hostname=" + hostname);
+ continue;
+ }
+
+ /*
+ * Add to the map.
+ *
+ * Note: We are adding the childNode. This is the CounterSet for the
+ * specific host. This means that the IHostScoringRules do not need
+ * to be aware of the hostname on which they are running. (The other
+ * approach would be to pass in the hostname as a prefix to the
+ * wrapper class that we are placing into the hostMetricsMap.)
+ */
hostMetricsMap.put(hostname, new CounterSetHostMetricsWrapper(
- counterSet));
+ (CounterSet) childNode));
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-05-14 17:09:04
|
Revision: 8319
http://sourceforge.net/p/bigdata/code/8319
Author: tobycraig
Date: 2014-05-14 17:09:02 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Highlight errors in queries/updates
Modified Paths:
--------------
branches/NEW_WORKBENCH_1_3_2_BRANCH/bigdata-war/src/html/css/style.css
branches/NEW_WORKBENCH_1_3_2_BRANCH/bigdata-war/src/html/js/workbench.js
Modified: branches/NEW_WORKBENCH_1_3_2_BRANCH/bigdata-war/src/html/css/style.css
===================================================================
--- branches/NEW_WORKBENCH_1_3_2_BRANCH/bigdata-war/src/html/css/style.css 2014-05-14 17:01:37 UTC (rev 8318)
+++ branches/NEW_WORKBENCH_1_3_2_BRANCH/bigdata-war/src/html/css/style.css 2014-05-14 17:09:02 UTC (rev 8319)
@@ -280,6 +280,19 @@
font-family: monospace;
}
+/* make cursor visible in error highlighting */
+div.CodeMirror-cursors {
+ z-index: 3;
+}
+
+.error-line {
+ background: red;
+}
+
+.error-character {
+ background: green;
+}
+
#page-selector {
float: right;
}
Modified: branches/NEW_WORKBENCH_1_3_2_BRANCH/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/NEW_WORKBENCH_1_3_2_BRANCH/bigdata-war/src/html/js/workbench.js 2014-05-14 17:01:37 UTC (rev 8318)
+++ branches/NEW_WORKBENCH_1_3_2_BRANCH/bigdata-war/src/html/js/workbench.js 2014-05-14 17:09:02 UTC (rev 8319)
@@ -2,9 +2,15 @@
// global variables
var DEFAULT_NAMESPACE, NAMESPACE, NAMESPACE_URL, NAMESPACES_READY, NAMESPACE_SHORTCUTS, FILE_CONTENTS, QUERY_RESULTS;
-var QUERY_EDITOR, UPDATE_EDITOR;
+var CODEMIRROR_DEFAULTS, EDITORS = {}, ERROR_LINE_MARKERS = {}, ERROR_CHARACTER_MARKERS = {};
var PAGE_SIZE = 50, TOTAL_PAGES, CURRENT_PAGE;
+CODEMIRROR_DEFAULTS = {
+ lineNumbers: true,
+ mode: 'sparql',
+ extraKeys: {'Ctrl-,': moveTabLeft, 'Ctrl-.': moveTabRight}
+};
+
// debug to access closure variables
$('html, textarea, select').bind('keydown', 'ctrl+d', function() { debugger; });
@@ -49,10 +55,8 @@
if(!nohash && window.location.hash.substring(1).indexOf(tab) != 0) {
window.location.hash = tab;
}
- if(tab == 'query') {
- QUERY_EDITOR.refresh();
- } else if(tab == 'update') {
- UPDATE_EDITOR.refresh();
+ if(EDITORS[tab]) {
+ EDITORS[tab].refresh();
}
}
@@ -379,7 +383,7 @@
mode = rdf_modes[type];
}
}
- UPDATE_EDITOR.setOption('mode', mode);
+ EDITORS.update.setOption('mode', mode);
}
// .xml is used for both RDF and TriX, assume it's RDF
@@ -420,9 +424,14 @@
$('#update-update').click(submitUpdate);
-UPDATE_EDITOR = CodeMirror.fromTextArea($('#update-box')[0], {lineNumbers: true, mode: 'sparql',
- extraKeys: {'Ctrl-Enter': submitUpdate, 'Ctrl-,': moveTabLeft, 'Ctrl-.': moveTabRight}
+EDITORS.update = CodeMirror.fromTextArea($('#update-box')[0], CODEMIRROR_DEFAULTS);
+EDITORS.update.on('change', function() {
+ if(ERROR_LINE_MARKERS.update) {
+ ERROR_LINE_MARKERS.update.clear();
+ ERROR_CHARACTER_MARKERS.update.clear();
+ }
});
+EDITORS.update.addKeyMap({'Ctrl-Enter': submitUpdate});
function submitUpdate(e) {
// Updates are submitted as a regular form for SPARQL updates in monitor mode, and via AJAX for non-monitor SPARQL, RDF & file path updates.
@@ -437,7 +446,7 @@
var settings = {
type: 'POST',
- data: FILE_CONTENTS == null ? UPDATE_EDITOR.getValue() : FILE_CONTENTS,
+ data: FILE_CONTENTS == null ? EDITORS.update.getValue() : FILE_CONTENTS,
success: updateResponseXML,
error: updateResponseError
}
@@ -538,9 +547,14 @@
}
});
-QUERY_EDITOR = CodeMirror.fromTextArea($('#query-box')[0], {lineNumbers: true, mode: 'sparql',
- extraKeys: {'Ctrl-Enter': submitQuery, 'Ctrl-,': moveTabLeft, 'Ctrl-.': moveTabRight}
+EDITORS.query = CodeMirror.fromTextArea($('#query-box')[0], CODEMIRROR_DEFAULTS);
+EDITORS.query.on('change', function() {
+ if(ERROR_LINE_MARKERS.query) {
+ ERROR_LINE_MARKERS.query.clear();
+ ERROR_CHARACTER_MARKERS.query.clear();
+ }
});
+EDITORS.query.addKeyMap({'Ctrl-Enter': submitQuery});
function submitQuery(e) {
try {
@@ -548,8 +562,13 @@
} catch(e) {}
// transfer CodeMirror content to textarea
- QUERY_EDITOR.save();
+ EDITORS.query.save();
+ // do nothing if query is empty
+ if($('#query-box').val().trim() == '') {
+ return;
+ }
+
var settings = {
type: 'POST',
data: $('#query-form').serialize(),
@@ -811,21 +830,9 @@
if(match) {
// highlight character at error position
var line = match[1] - 1;
- var column = match[2] - 1;
- var input = $('#' + pane + '-box').val();
- var lines = input.split('\n');
- var container = '#' + pane + '-errors';
- $(container).html('');
- for(var i=0; i<line; i++) {
- var p = $('<p>').text(lines[i]);
- $(container).append(p);
- }
- $(container).append('<p class="error-line">');
- $(container + ' .error-line').append(document.createTextNode(lines[line].substr(0, column)));
- $(container + ' .error-line').append($('<span class="error-character">').text(lines[line].charAt(column) || ' '));
- $(container + ' .error-line').append(document.createTextNode(lines[line].substr(column + 1)));
- $(container).show();
- $('#' + pane + '-box').scrollTop(0);
+ var character = match[2] - 1;
+ ERROR_LINE_MARKERS[pane] = EDITORS.query.doc.markText({line: line, ch: 0}, {line: line}, {className: 'error-line'});
+ ERROR_CHARACTER_MARKERS[pane] = EDITORS.query.doc.markText({line: line, ch: character}, {line: line, ch: character + 1}, {className: 'error-character'});
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 17:01:40
|
Revision: 8318
http://sourceforge.net/p/bigdata/code/8318
Author: thompsonbry
Date: 2014-05-14 17:01:37 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Bug fixes to linux performance counter collectors.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/linux/PIDStatCollector.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/linux/SarCpuUtilizationCollector.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/linux/VMStatCollector.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/linux/PIDStatCollector.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/linux/PIDStatCollector.java 2014-05-14 16:44:47 UTC (rev 8317)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/linux/PIDStatCollector.java 2014-05-14 17:01:37 UTC (rev 8318)
@@ -28,13 +28,12 @@
package com.bigdata.counters.linux;
-import java.io.File;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
import com.bigdata.counters.AbstractProcessCollector;
import com.bigdata.counters.AbstractProcessReader;
@@ -61,7 +60,6 @@
* repeat forever if interval was specified.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class PIDStatCollector extends AbstractProcessCollector implements
ICounterHierarchy, IProcessCounters {
@@ -92,7 +90,6 @@
* hierarchy.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
abstract class AbstractInst<T> implements IInstrument<T> {
@@ -104,17 +101,19 @@
}
- protected AbstractInst(String path) {
+ protected AbstractInst(final String path) {
+
+ if (path == null)
+ throw new IllegalArgumentException();
- assert path != null;
-
this.path = path;
}
+ @Override
final public long lastModified() {
- return lastModified;
+ return lastModified.get();
}
@@ -122,7 +121,8 @@
* @throws UnsupportedOperationException
* always.
*/
- final public void setValue(T value, long timestamp) {
+ @Override
+ final public void setValue(final T value, final long timestamp) {
throw new UnsupportedOperationException();
@@ -135,13 +135,12 @@
* hierarchy.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
class IL extends AbstractInst<Long> {
protected final long scale;
- public IL(String path, long scale) {
+ public IL(final String path, final long scale) {
super( path );
@@ -149,6 +148,7 @@
}
+ @Override
public Long getValue() {
final Long value = (Long) vals.get(path);
@@ -170,13 +170,12 @@
* hierarchy.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
class ID extends AbstractInst<Double> {
protected final double scale;
- public ID(String path, double scale) {
+ public ID(final String path, final double scale) {
super(path);
@@ -184,6 +183,7 @@
}
+ @Override
public Double getValue() {
final Double value = (Double) vals.get(path);
@@ -205,7 +205,7 @@
* as the last modified time for counters based on that process and
* defaulted to the time that we begin to collect performance data.
*/
- private long lastModified = System.currentTimeMillis();
+ private final AtomicLong lastModified = new AtomicLong(System.currentTimeMillis());
/**
* Map containing the current values for the configured counters. The keys
@@ -217,7 +217,7 @@
* declared within {@link #getCounters()} and not whatever path the counters
* are eventually placed under within a larger hierarchy.
*/
- private Map<String,Object> vals = new HashMap<String, Object>();
+ private final Map<String,Object> vals = new ConcurrentHashMap<String, Object>();
/**
* @param pid
@@ -229,7 +229,8 @@
*
* @todo kernelVersion could be static.
*/
- public PIDStatCollector(int pid, int interval, KernelVersion kernelVersion) {
+ public PIDStatCollector(final int pid, final int interval,
+ final KernelVersion kernelVersion) {
super(interval);
@@ -269,79 +270,69 @@
command.add("-r"); // memory report
// command.add("-w"); // context switching report (not implemented in our code).
-
- command.add(""+getInterval());
-
+
+ command.add("" + getInterval());
+
return command;
}
- /**
- * Declare the counters that we will collect using <code>pidstat</code>.
- * These counters are NOT placed within the counter hierarchy but are
- * declared using the bare path for the counter. E.g., as
- * {@link IProcessCounters#Memory_virtualSize}.
- */
- /*synchronized*/ public CounterSet getCounters() {
+ @Override
+ public CounterSet getCounters() {
-// if(root == null) {
-
- final CounterSet root = new CounterSet();
-
- inst = new LinkedList<AbstractInst<?>>();
-
- /*
- * Note: Counters are all declared as Double to facilitate
- * aggregation and scaling.
- *
- * Note: pidstat reports percentages as [0:100] so we normalize them
- * to [0:1] using a scaling factor.
- */
+ final List<AbstractInst<?>> inst = new LinkedList<AbstractInst<?>>();
- inst.add(new ID(IProcessCounters.CPU_PercentUserTime,.01d));
- inst.add(new ID(IProcessCounters.CPU_PercentSystemTime,.01d));
- inst.add(new ID(IProcessCounters.CPU_PercentProcessorTime,.01d));
-
- inst.add(new ID(IProcessCounters.Memory_minorFaultsPerSec,1d));
- inst.add(new ID(IProcessCounters.Memory_majorFaultsPerSec,1d));
- inst.add(new IL(IProcessCounters.Memory_virtualSize,Bytes.kilobyte));
- inst.add(new IL(IProcessCounters.Memory_residentSetSize,Bytes.kilobyte));
- inst.add(new ID(IProcessCounters.Memory_percentMemorySize,.01d));
+ /*
+ * Note: Counters are all declared as Double to facilitate aggregation
+ * and scaling.
+ *
+ * Note: pidstat reports percentages as [0:100] so we normalize them to
+ * [0:1] using a scaling factor.
+ */
- /*
- * Note: pidstat reports in kb/sec so we normalize to bytes/second
- * using a scaling factor.
- */
- inst.add(new ID(IProcessCounters.PhysicalDisk_BytesReadPerSec, Bytes.kilobyte32));
- inst.add(new ID(IProcessCounters.PhysicalDisk_BytesWrittenPerSec, Bytes.kilobyte32));
+ inst.add(new ID(IProcessCounters.CPU_PercentUserTime, .01d));
+ inst.add(new ID(IProcessCounters.CPU_PercentSystemTime, .01d));
+ inst.add(new ID(IProcessCounters.CPU_PercentProcessorTime, .01d));
-// }
+ inst.add(new ID(IProcessCounters.Memory_minorFaultsPerSec, 1d));
+ inst.add(new ID(IProcessCounters.Memory_majorFaultsPerSec, 1d));
+ inst.add(new IL(IProcessCounters.Memory_virtualSize, Bytes.kilobyte));
+ inst.add(new IL(IProcessCounters.Memory_residentSetSize, Bytes.kilobyte));
+ inst.add(new ID(IProcessCounters.Memory_percentMemorySize, .01d));
+
+ /*
+ * Note: pidstat reports in kb/sec so we normalize to bytes/second using
+ * a scaling factor.
+ */
+ inst.add(new ID(IProcessCounters.PhysicalDisk_BytesReadPerSec,
+ Bytes.kilobyte32));
+ inst.add(new ID(IProcessCounters.PhysicalDisk_BytesWrittenPerSec,
+ Bytes.kilobyte32));
+
+ final CounterSet root = new CounterSet();
- for(Iterator<AbstractInst<?>> itr = inst.iterator(); itr.hasNext(); ) {
-
- final AbstractInst<?> i = itr.next();
-
+ for (AbstractInst<?> i : inst) {
+
root.addCounter(i.getPath(), i);
-
+
}
-
+
return root;
-
+
}
- private List<AbstractInst<?>> inst = null;
-// private CounterSet root = null;
-
+
/**
- * Extended to force <code>pidstat</code> to use a consistent
- * timestamp format regardless of locale by setting
- * <code>S_TIME_FORMAT="ISO"</code> in the environment.
+ * Extended to force <code>pidstat</code> to use a consistent timestamp
+ * format regardless of locale by setting <code>S_TIME_FORMAT="ISO"</code>
+ * in the environment.
*/
- protected void setEnvironment(Map<String, String> env) {
+ @Override
+ protected void setEnvironment(final Map<String, String> env) {
super.setEnvironment(env);
-
+
env.put("S_TIME_FORMAT", "ISO");
-
+
}
@Override
@@ -374,10 +365,10 @@
* </pre>
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
protected class PIDStatReader extends ProcessReaderHelper {
+ @Override
protected ActiveProcess getActiveProcess() {
if (activeProcess == null)
@@ -410,6 +401,7 @@
* is possible that this will not work when the host is
* using an English locale.
*/
+ @Override
protected void readProcess() throws IOException, InterruptedException {
if(log.isInfoEnabled())
@@ -478,7 +470,7 @@
* time of the start of the current day, which is what we would have
* to do.
*/
- lastModified = System.currentTimeMillis();
+ lastModified.set(System.currentTimeMillis());
if(header.contains("%CPU")) {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/linux/SarCpuUtilizationCollector.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/linux/SarCpuUtilizationCollector.java 2014-05-14 16:44:47 UTC (rev 8317)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/linux/SarCpuUtilizationCollector.java 2014-05-14 17:01:37 UTC (rev 8318)
@@ -28,12 +28,11 @@
package com.bigdata.counters.linux;
-import java.io.File;
-import java.util.HashMap;
-import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
import com.bigdata.counters.AbstractProcessCollector;
import com.bigdata.counters.AbstractProcessReader;
@@ -51,31 +50,16 @@
* <code>sar -u</code>.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class SarCpuUtilizationCollector extends AbstractProcessCollector
implements ICounterHierarchy, IRequiredHostCounters, IHostCounters {
-// static protected final Logger log = Logger
-// .getLogger(SarCpuUtilizationCollector.class);
-//
-// /**
-// * True iff the {@link #log} level is DEBUG or less.
-// */
-// final protected static boolean DEBUG = log.isDebugEnabled();
-//
-// /**
-// * True iff the {@link #log} level is log.isInfoEnabled() or less.
-// */
-// final protected static boolean log.isInfoEnabled() = log.isInfoEnabled();
-
/**
* Inner class integrating the current values with the {@link ICounterSet}
* hierarchy.
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*/
abstract class I<T> implements IInstrument<T> {
@@ -87,17 +71,19 @@
}
- public I(String path) {
+ public I(final String path) {
- assert path != null;
+ if (path == null)
+ throw new IllegalArgumentException();
this.path = path;
}
+ @Override
public long lastModified() {
- return lastModified;
+ return lastModified.get();
}
@@ -105,7 +91,8 @@
* @throws UnsupportedOperationException
* always.
*/
- public void setValue(T value, long timestamp) {
+ @Override
+ public void setValue(final T value, final long timestamp) {
throw new UnsupportedOperationException();
@@ -117,13 +104,12 @@
* Double precision counter with scaling factor.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
class DI extends I<Double> {
protected final double scale;
- DI(String path, double scale) {
+ DI(final String path, final double scale) {
super( path );
@@ -131,7 +117,7 @@
}
-
+ @Override
public Double getValue() {
final Double value = (Double) vals.get(path);
@@ -153,12 +139,12 @@
* keys are paths into the {@link CounterSet}. The values are the data
* most recently read from <code>sar</code>.
*/
- private Map<String,Object> vals = new HashMap<String, Object>();
+ private final Map<String,Object> vals = new ConcurrentHashMap<String, Object>();
/**
* The timestamp associated with the most recently collected values.
*/
- private long lastModified = System.currentTimeMillis();
+ private final AtomicLong lastModified = new AtomicLong(System.currentTimeMillis());
/**
*
@@ -173,6 +159,7 @@
}
+ @Override
public List<String> getCommand() {
final List<String> command = new LinkedList<String>();
@@ -192,53 +179,44 @@
}
- /**
- * Declares the counters that we will collect using <code>sar</code>.
- */
- /*synchronized*/ public CounterSet getCounters() {
+ @Override
+ public CounterSet getCounters() {
-// if(root == null) {
-
- final CounterSet root = new CounterSet();
-
- inst = new LinkedList<I>();
-
- /*
- * Note: Counters are all declared as Double to facilitate
- * aggregation.
- *
- * Note: sar reports percentages in [0:100] so we convert them to
- * [0:1] using a scaling factor.
- */
+ final CounterSet root = new CounterSet();
- inst.add(new DI(IRequiredHostCounters.CPU_PercentProcessorTime,.01d));
-
- inst.add(new DI(IHostCounters.CPU_PercentUserTime,.01d));
- inst.add(new DI(IHostCounters.CPU_PercentSystemTime,.01d));
- inst.add(new DI(IHostCounters.CPU_PercentIOWait,.01d));
-
- for(Iterator<I> itr = inst.iterator(); itr.hasNext(); ) {
-
- final I i = itr.next();
-
- root.addCounter(i.getPath(), i);
-
- }
-
-// }
+ @SuppressWarnings("rawtypes")
+ final List<I> inst = new LinkedList<I>();
+
+ /*
+ * Note: Counters are all declared as Double to facilitate aggregation.
+ *
+ * Note: sar reports percentages in [0:100] so we convert them to [0:1]
+ * using a scaling factor.
+ */
+
+ inst.add(new DI(IRequiredHostCounters.CPU_PercentProcessorTime, .01d));
+
+ inst.add(new DI(IHostCounters.CPU_PercentUserTime, .01d));
+ inst.add(new DI(IHostCounters.CPU_PercentSystemTime, .01d));
+ inst.add(new DI(IHostCounters.CPU_PercentIOWait, .01d));
+
+ for (@SuppressWarnings("rawtypes") I i : inst) {
+
+ root.addCounter(i.getPath(), i);
+
+ }
return root;
}
- private List<I> inst = null;
-// private CounterSet root = null;
/**
* Extended to force <code>sar</code> to use a consistent timestamp
* format regardless of locale by setting
* <code>S_TIME_FORMAT="ISO"</code> in the environment.
*/
- protected void setEnvironment(Map<String, String> env) {
+ @Override
+ protected void setEnvironment(final Map<String, String> env) {
super.setEnvironment(env);
@@ -246,6 +224,7 @@
}
+ @Override
public AbstractProcessReader getProcessReader() {
return new SarReader();
@@ -269,10 +248,10 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*/
private class SarReader extends ProcessReaderHelper {
+ @Override
protected ActiveProcess getActiveProcess() {
if (activeProcess == null)
@@ -376,7 +355,7 @@
* adjusting for the UTC time of the start of the current day,
* which is what we would have to do.
*/
- lastModified = System.currentTimeMillis();
+ lastModified.set(System.currentTimeMillis());
// final String user = data.substring(20-1, 30-1);
//// final String nice = data.substring(30-1, 40-1);
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/linux/VMStatCollector.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/linux/VMStatCollector.java 2014-05-14 16:44:47 UTC (rev 8317)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/linux/VMStatCollector.java 2014-05-14 17:01:37 UTC (rev 8318)
@@ -28,11 +28,11 @@
package com.bigdata.counters.linux;
-import java.util.HashMap;
-import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
import com.bigdata.counters.AbstractProcessCollector;
@@ -50,7 +50,6 @@
* Collects some counters using <code>vmstat</code>.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class VMStatCollector extends AbstractProcessCollector implements
ICounterHierarchy, IRequiredHostCounters, IHostCounters{
@@ -61,7 +60,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*/
abstract class I<T> implements IInstrument<T> {
@@ -73,17 +71,19 @@
}
- public I(String path) {
+ public I(final String path) {
- assert path != null;
+ if (path == null)
+ throw new IllegalArgumentException();
this.path = path;
}
+ @Override
public long lastModified() {
- return lastModified;
+ return lastModified.get();
}
@@ -91,7 +91,8 @@
* @throws UnsupportedOperationException
* always.
*/
- public void setValue(T value, long timestamp) {
+ @Override
+ public void setValue(final T value, final long timestamp) {
throw new UnsupportedOperationException();
@@ -108,15 +109,15 @@
protected final double scale;
- DI(String path, double scale) {
+ DI(final String path, final double scale) {
+
+ super(path);
- super( path );
-
this.scale = scale;
}
-
+ @Override
public Double getValue() {
final Double value = (Double) vals.get(path);
@@ -138,24 +139,26 @@
* are paths into the {@link CounterSet}. The values are the data most
* recently read from <code>vmstat</code>.
*/
- final private Map<String, Object> vals = new HashMap<String, Object>();
+ final private Map<String, Object> vals = new ConcurrentHashMap<String, Object>();
/**
* The timestamp associated with the most recently collected values.
*/
- private long lastModified = System.currentTimeMillis();
+ private final AtomicLong lastModified = new AtomicLong(
+ System.currentTimeMillis());
/**
* <code>true</code> iff you want collect the user time, system time,
* and IO WAIT time using vmstat (as opposed to sar).
*/
- protected final boolean cpuStats;
+ private final boolean cpuStats;
/**
* The {@link Pattern} used to split apart the rows read from
* <code>vmstat</code>.
*/
- final protected static Pattern pattern = Pattern.compile("\\s+");
+ // Note: Exposed to the test suite.
+ final static Pattern pattern = Pattern.compile("\\s+");
/**
*
@@ -173,6 +176,7 @@
}
+ @Override
public List<String> getCommand() {
final List<String> command = new LinkedList<String>();
@@ -192,12 +196,12 @@
/**
* Declares the counters that we will collect
*/
+ @Override
public CounterSet getCounters() {
- final CounterSet root = new CounterSet();
+ @SuppressWarnings("rawtypes")
+ final List<I> inst = new LinkedList<I>();
- inst = new LinkedList<I>();
-
/*
* Note: Counters are all declared as Double to facilitate aggregation.
*/
@@ -257,20 +261,19 @@
}
- for (Iterator<I> itr = inst.iterator(); itr.hasNext();) {
+ final CounterSet root = new CounterSet();
- final I i = itr.next();
+ for (@SuppressWarnings("rawtypes") I i : inst) {
- root.addCounter(i.getPath(), i);
+ root.addCounter(i.getPath(), i);
- }
+ }
return root;
}
- private List<I> inst = null;
-
+ @Override
public AbstractProcessReader getProcessReader() {
return new VMStatReader();
@@ -296,6 +299,7 @@
*/
private class VMStatReader extends ProcessReaderHelper {
+ @Override
protected ActiveProcess getActiveProcess() {
if (activeProcess == null)
@@ -317,7 +321,7 @@
if(log.isInfoEnabled())
log.info("begin");
- for(int i=0; i<10 && !getActiveProcess().isAlive(); i++) {
+ for (int i = 0; i < 10 && !getActiveProcess().isAlive(); i++) {
if(log.isInfoEnabled())
log.info("waiting for the readerFuture to be set.");
@@ -362,7 +366,7 @@
try {
// timestamp
- lastModified = System.currentTimeMillis();
+ lastModified.set(System.currentTimeMillis());
final String[] fields = pattern.split(data.trim(), 0/* limit */);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-05-14 16:44:50
|
Revision: 8317
http://sourceforge.net/p/bigdata/code/8317
Author: tobycraig
Date: 2014-05-14 16:44:47 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Recreated workbench branch
Added Paths:
-----------
branches/NEW_WORKBENCH_1_3_2_BRANCH/
Index: branches/NEW_WORKBENCH_1_3_2_BRANCH
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0 2014-05-14 16:19:55 UTC (rev 8316)
+++ branches/NEW_WORKBENCH_1_3_2_BRANCH 2014-05-14 16:44:47 UTC (rev 8317)
Property changes on: branches/NEW_WORKBENCH_1_3_2_BRANCH
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,31 ##
+ant-build
+src
+bin
+bigdata*.jar
+ant-release
+standalone
+test*
+countersfinal.xml
+events.jnl
+.settings
+*.jnl
+TestInsertRate.out
+SYSTAP-BBT-result.txt
+U10load+query
+*.hprof
+com.bigdata.cache.TestHardReferenceQueueWithBatchingUpdates.exp.csv
+commit-log.txt
+eventLog
+dist
+bigdata-test
+com.bigdata.rdf.stress.LoadClosureAndQueryTest.*.csv
+DIST.bigdata-*.tgz
+REL.bigdata-*.tgz
+queryLog*
+queryRunState*
+sparql.txt
+benchmark
+CI
+bsbm10-dataset.nt.gz
+bsbm10-dataset.nt.zip
+benchmark*
Added: svn:mergeinfo
## -0,0 +1,20 ##
+/branches/BIGDATA_MGC_HA1_HA5:8025-8122
+/branches/BIGDATA_OPENRDF_2_6_9_UPDATE:6769-6785
+/branches/BIGDATA_RELEASE_1_2_0:6766-7380
+/branches/BTREE_BUFFER_BRANCH:2004-2045
+/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
+/branches/INT64_BRANCH:4486-4522
+/branches/JOURNAL_HA_BRANCH:2596-4066
+/branches/LARGE_LITERALS_REFACTOR:4175-4387
+/branches/LEXICON_REFACTOR_BRANCH:2633-3304
+/branches/MGC_1_3_0:7609-7752
+/branches/QUADS_QUERY_BRANCH:4525-4531,4550-4584,4586-4609,4634-4643,4646-4672,4674-4685,4687-4693,4697-4735,4737-4782,4784-4792,4794-4796,4798-4801
+/branches/RDR:7665-8159
+/branches/READ_CACHE:7215-7271
+/branches/RWSTORE_1_1_0_DEBUG:5896-5935
+/branches/TIDS_PLUS_BLOBS_BRANCH:4814-4836
+/branches/ZK_DISCONNECT_HANDLING:7465-7484
+/branches/bugfix-btm:2594-3237
+/branches/dev-btm:2574-2730
+/branches/fko:3150-3194
+/trunk:3392-3437,3656-4061
\ No newline at end of property
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 16:20:00
|
Revision: 8316
http://sourceforge.net/p/bigdata/code/8316
Author: thompsonbry
Date: 2014-05-14 16:19:55 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Modifying log message to track down contexts in which the CounterSet is refusing the replace of a performance counter. See #624 (LBS)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/CounterSet.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/CounterSet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/CounterSet.java 2014-05-14 15:48:14 UTC (rev 8315)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/counters/CounterSet.java 2014-05-14 16:19:55 UTC (rev 8316)
@@ -44,6 +44,8 @@
import org.apache.log4j.Logger;
import org.xml.sax.SAXException;
+import com.bigdata.util.StackInfoReport;
+
import cutthecrap.utils.striterators.Expander;
import cutthecrap.utils.striterators.Filter;
import cutthecrap.utils.striterators.IStriterator;
@@ -798,7 +800,7 @@
if(counter instanceof ICounter ) {
// counter exists for that path.
- log.error("Exists: path=" + getPath() + ", name=" + name);
+ log.error(new StackInfoReport("Exists: path=" + getPath() + ", name=" + name));
// return existing counter for path @todo vs replace.
return (ICounter)counter;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 15:48:19
|
Revision: 8315
http://sourceforge.net/p/bigdata/code/8315
Author: thompsonbry
Date: 2014-05-14 15:48:14 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Added worksheet for the HA LBS LOAD => AVAILABILITY normalization logic and link to worksheet in the HA LBS code. See #624 (HA LBS)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/architecture/HA_LBS.xlsx
Added: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/architecture/HA_LBS.xlsx
===================================================================
(Binary files differ)
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/architecture/HA_LBS.xlsx
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/architecture/HA_LBS.xlsx 2014-05-14 15:32:55 UTC (rev 8314)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/architecture/HA_LBS.xlsx 2014-05-14 15:48:14 UTC (rev 8315)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/architecture/HA_LBS.xlsx
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+application/octet-stream
\ No newline at end of property
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 15:32:55 UTC (rev 8314)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 15:48:14 UTC (rev 8315)
@@ -762,6 +762,8 @@
* be proxied -or- <code>null</code> if the request should not be
* proxied (because we lack enough information to identify a target
* host).
+ *
+ * @see bigdata/src/resources/architecture/HA_LBS.xls
*/
static HostScore getHost(//
final double d, //
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 15:32:59
|
Revision: 8314
http://sourceforge.net/p/bigdata/code/8314
Author: thompsonbry
Date: 2014-05-14 15:32:55 +0000 (Wed, 14 May 2014)
Log Message:
-----------
LBS policy fix.
Moved the JVM_OPTS into /etc/defaults/bigdataHA. This is the more standard practice.
See #624 (HA LBS)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices
branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 15:16:27 UTC (rev 8313)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 15:32:55 UTC (rev 8314)
@@ -802,7 +802,7 @@
double sum = 0d;
for (HostScore tmp : hostScores) {
hostScore = tmp;
- sum += (1d - hostScore.getScore());
+ sum += hostScore.getScore();
if (sum >= d) {
// found desired host.
break;
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices 2014-05-14 15:16:27 UTC (rev 8313)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices 2014-05-14 15:32:55 UTC (rev 8314)
@@ -21,17 +21,6 @@
pidFile=$lockDir/pid
##
-# ServiceStarter JVM options.
-#
-# The ServiceStarter is launched as a JVM with the following JVM options.
-# The other services (including the HAJournalServer) will run inside of
-# this JVM. This is where you specify the size of the Java heap and the
-# size of the direct memory heap (used for the write cache buffers and
-# some related things).
-##
-export JVM_OPTS="-server -Xmx4G -XX:MaxDirectMemorySize=3000m"
-
-##
# HAJournalServer configuration parameter overrides (see HAJournal.config).
#
# The bigdata HAJournal.config file may be heavily parameterized through
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA 2014-05-14 15:16:27 UTC (rev 8313)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA 2014-05-14 15:32:55 UTC (rev 8314)
@@ -14,6 +14,18 @@
#pidFile=
##
+# ServiceStarter JVM options.
+#
+# The ServiceStarter is launched as a JVM with the following JVM options.
+# The other services (including the HAJournalServer) will run inside of
+# this JVM. This is where you specify the size of the Java heap and the
+# size of the direct memory heap (used for the write cache buffers and
+# some related things).
+##
+export JVM_OPTS="-server -Xmx4G -XX:MaxDirectMemorySize=3000m"
+#export JVM_OPTS="-server -Xmx4G -XX:MaxDirectMemorySize=3000m -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=1046"
+
+##
# The following variables configure the startHAServices script, which
# passes them through to HAJournal.config.
##
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 15:16:30
|
Revision: 8313
http://sourceforge.net/p/bigdata/code/8313
Author: thompsonbry
Date: 2014-05-14 15:16:27 +0000 (Wed, 14 May 2014)
Log Message:
-----------
LBS logic fix.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 15:02:17 UTC (rev 8312)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 15:16:27 UTC (rev 8313)
@@ -609,7 +609,7 @@
{
for (int i = 0; i < nhosts; i++) {
- final double avail = totalLoad - load[i];
+ final double avail = availability[i] = totalLoad - load[i];
totalAvailability += avail;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 15:02:21
|
Revision: 8312
http://sourceforge.net/p/bigdata/code/8312
Author: thompsonbry
Date: 2014-05-14 15:02:17 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Still working on the LBS host scoring logic.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/lbs/TestAbstractHostLBSPolicy.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 14:16:19 UTC (rev 8311)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 15:02:17 UTC (rev 8312)
@@ -539,9 +539,9 @@
final IHostMetrics[] metrics2 = new IHostMetrics[nhosts];
- final double[] hostScores = new double[nhosts];
+ final double[] load = new double[nhosts];
- double totalScore = 0d;
+ double totalLoad = 0d;
{
@@ -583,30 +583,40 @@
hostnames[i] = hostname;
- hostScores[i] = hostScore;
+ load[i] = hostScore;
metrics2[i] = metrics;
- totalScore += hostScore;
+ totalLoad += hostScore;
i++;
}
-
- if (totalScore == 0) {
-
- /*
- * If totalScore is zero, then weight all hosts equally as
- * (1/nhosts).
- */
-
- totalScore = nhosts;
-
- }
}
/*
+ * Convert from LOAD to AVAILABILITY.
+ *
+ * AVAILABILITY := TOTAL - LOAD[i]
+ *
+ * Note: The per-host metrics and scoring rule give us LOAD. However, we
+ * want to distribute the requests based on the inverse of the load,
+ * which is the AVAILABILITY to do more work.
+ */
+ double totalAvailability = 0;
+ double availability[] = new double[nhosts];
+ {
+ for (int i = 0; i < nhosts; i++) {
+
+ final double avail = totalLoad - load[i];
+
+ totalAvailability += avail;
+
+ }
+ }
+
+ /*
* Normalize the per-hosts scores.
*/
@@ -614,13 +624,22 @@
final HostScore[] scores = new HostScore[nhosts];
{
- for (int i = 0; i < scores.length; i++) {
+ for (int i = 0; i < nhosts; i++) {
final String hostname = hostnames[i];
+ final double normalizedAvailability;
+ if (totalAvailability == 0) {
+ // divide the work evenly.
+ normalizedAvailability = 1d / nhosts;
+ } else {
+ normalizedAvailability = availability[i]
+ / totalAvailability;
+ }
+
// Normalize host scores.
final HostScore hostScore = scores[i] = new HostScore(hostname,
- hostScores[i], totalScore, metrics2[i], scoringRule);
+ normalizedAvailability);
if (thisHostScore != null && hostScore.isThisHost()) {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java 2014-05-14 14:16:19 UTC (rev 8311)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java 2014-05-14 15:02:17 UTC (rev 8312)
@@ -41,12 +41,12 @@
*/
private final boolean thisHost;
- /**
- * The raw score for some host. This is a measure of the load on a host. The
- * measure is computed based on {@link #metrics} using the
- * {@link #scoringRule}.
- */
- private final double rawScore;
+// /**
+// * The raw score for some host. This is a measure of the load on a host. The
+// * measure is computed based on {@link #metrics} using the
+// * {@link #scoringRule}.
+// */
+// private final double rawScore;
/**
* The normalized score for that host.
@@ -59,18 +59,18 @@
*/
private final double score;
- /**
- * The {@link IHostScoringRule} used to convert the {@link #metrics} into
- * the {@link #rawScore}.
- */
- private final IHostScoringRule scoringRule;
-
- /**
- * The {@link IHostMetrics} associated with this host (this is
- * informational. The {@link #metrics} provide the detailed per-host
- * performance metrics that were intepreted by the {@link #scoringRule} .
- */
- final private IHostMetrics metrics;
+// /**
+// * The {@link IHostScoringRule} used to convert the {@link #metrics} into
+// * the {@link #rawScore}.
+// */
+// private final IHostScoringRule scoringRule;
+//
+// /**
+// * The {@link IHostMetrics} associated with this host (this is
+// * informational. The {@link #metrics} provide the detailed per-host
+// * performance metrics that were intepreted by the {@link #scoringRule} .
+// */
+// final private IHostMetrics metrics;
// /** The rank in [0:#scored]. This is an index into the Scores[]. */
// public int rank = -1;
@@ -78,14 +78,14 @@
// /** The normalized double precision rank in [0.0:1.0]. */
// public double drank = -1d;
- /**
- * Return the raw score (aka load) for a host. This raw score is an
- * unnormalized measure of the load on that host. The measure is computed
- * based on {@link IHostMetrics} using some {@link IHostScoringRule}.
- */
- public double getRawScore() {
- return rawScore;
- }
+// /**
+// * Return the raw score (aka load) for a host. This raw score is an
+// * unnormalized measure of the load on that host. The measure is computed
+// * based on {@link IHostMetrics} using some {@link IHostScoringRule}.
+// */
+// public double getRawScore() {
+// return rawScore;
+// }
/**
* Return the normalized load for the host.
@@ -105,22 +105,22 @@
return hostname;
}
- /**
- * The {@link IHostMetrics} associated with this host (optional). The
- * {@link #getMetrics()} provide the detailed per-host performance metrics
- * that were intepreted by the {@link #getScoringRule()} .
- */
- public IHostMetrics getMetrics() {
- return metrics;
- }
-
- /**
- * The {@link IHostScoringRule} used to convert the {@link #getMetrics()}
- * into the {@link #getRawScore()} (optional).
- */
- public IHostScoringRule getScoringRule() {
- return scoringRule;
- }
+// /**
+// * The {@link IHostMetrics} associated with this host (optional). The
+// * {@link #getMetrics()} provide the detailed per-host performance metrics
+// * that were intepreted by the {@link #getScoringRule()} .
+// */
+// public IHostMetrics getMetrics() {
+// return metrics;
+// }
+//
+// /**
+// * The {@link IHostScoringRule} used to convert the {@link #getMetrics()}
+// * into the {@link #getRawScore()} (optional).
+// */
+// public IHostScoringRule getScoringRule() {
+// return scoringRule;
+// }
/**
* Return <code>true</code> iff the host is this host.
@@ -135,12 +135,12 @@
return "HostScore"//
+ "{hostname=" + hostname //
+ ", thisHost=" + thisHost//
- + ", rawScore=" + rawScore //
+// + ", rawScore=" + rawScore //
+ ", score=" + score //
// + ", rank=" + rank //
// + ", drank=" + drank //
- + ", metrics=" + metrics //
- + ", scoringRule=" + scoringRule //
+// + ", metrics=" + metrics //
+// + ", scoringRule=" + scoringRule //
+ "}";
}
@@ -149,22 +149,25 @@
*
* @param hostname
* The hostname (required, must be non-empty).
- * @param rawScore
- * The unnormalized load for that host.
- * @param totalRawScore
- * The total unnormalized load across all hosts.
- * @param metrics
- * The performance metrics used to compute the unnormalized load
- * for each host (optional).
- * @param scoringRule
- * The rule used to score those metrics (optional).
+ * @param score The normalized availability score for this
+ * host.
+// * @param rawScore
+// * The unnormalized load for that host.
+// * @param totalRawScore
+// * The total unnormalized load across all hosts.
*/
+// * @param metrics
+// * The performance metrics used to compute the unnormalized load
+// * for each host (optional).
+// * @param scoringRule
+// * The rule used to score those metrics (optional).
public HostScore(//
final String hostname,//
- final double rawScore,//
- final double totalRawScore, //
- final IHostMetrics metrics,//
- final IHostScoringRule scoringRule//
+ final double score
+// final double rawScore,//
+// final double totalRawScore //
+// final IHostMetrics metrics,//
+// final IHostScoringRule scoringRule//
) {
if (hostname == null)
@@ -173,54 +176,59 @@
if (hostname.trim().length() == 0)
throw new IllegalArgumentException();
+ if (score < 0d || score > 1d)
+ throw new IllegalArgumentException();
+
this.hostname = hostname;
- this.rawScore = rawScore;
+ this.score = score;
+
+// this.rawScore = rawScore;
this.thisHost = AbstractStatisticsCollector.fullyQualifiedHostName
.equals(hostname);
- this.scoringRule = scoringRule;
+// this.scoringRule = scoringRule;
+//
+// this.metrics = metrics;
- this.metrics = metrics;
+// score = normalize(rawScore, totalRawScore);
- score = normalize(rawScore, totalRawScore);
-
}
- /**
- * Computes the normalized {@link #score} from the {@link #rawScore} in the
- * context of total over the {@link #rawScore}s for some set of hosts.
- *
- * @param rawScore
- * The raw score.
- * @param totalRawScore
- * The raw score computed from the totals.
- *
- * @return The normalized score.
- */
- static private double normalize(final double rawScore,
- final double totalRawScore) {
+// /**
+// * Computes the normalized {@link #score} from the {@link #rawScore} in the
+// * context of total over the {@link #rawScore}s for some set of hosts.
+// *
+// * @param rawScore
+// * The raw score.
+// * @param totalRawScore
+// * The raw score computed from the totals.
+// *
+// * @return The normalized score.
+// */
+// static private double normalize(final double rawScore,
+// final double totalRawScore) {
+//
+// if (totalRawScore == 0d) {
+//
+// return 0d;
+//
+// }
+//
+// final double score = rawScore / totalRawScore;
+//
+// if (score < 0 || score > 1) {
+//
+// throw new RuntimeException("score(" + score + ") := rawScore("
+// + rawScore + ") / totalRawScore(" + totalRawScore + ")");
+//
+// }
+//
+// return score;
+//
+// }
- if (totalRawScore == 0d) {
-
- return 0d;
-
- }
-
- final double score = rawScore / totalRawScore;
-
- if (score < 0 || score > 1) {
-
- throw new RuntimeException("score(" + score + ") := rawScore("
- + rawScore + ") / totalRawScore(" + totalRawScore + ")");
-
- }
-
- return score;
-
- }
-
/**
* Places elements into order by decreasing {@link #getScore() normalized
* load}. The {@link #getHostname()} is used to break any ties (but this
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/lbs/TestAbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/lbs/TestAbstractHostLBSPolicy.java 2014-05-14 14:16:19 UTC (rev 8311)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/lbs/TestAbstractHostLBSPolicy.java 2014-05-14 15:02:17 UTC (rev 8312)
@@ -22,7 +22,6 @@
*/
package com.bigdata.rdf.sail.webapp.lbs;
-import java.util.Arrays;
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
@@ -75,8 +74,7 @@
final String H1 = "H1";
// The load scores for those hosts.
- final HostScore hostScore1 = new HostScore("H1", .5d/* rawScore */,
- .5d/* totalRawScore */, null/* scoringRule */, null/* metrics */);
+ final HostScore hostScore1 = new HostScore("H1", 1d);
// should sum to 1.0
assertEquals(1d, hostScore1.getScore());
@@ -191,180 +189,177 @@
* This test covers the case with 3 hosts, each running one service.
*/
public void test_HA3() {
-
- /*
- * The load scores for those hosts.
- *
- * Note: These need to be ordered by increasing load. That is how they
- * are sorted by the LBS code. The running totals in terms of the
- * normalized load.
- */
- final double totalRawScore = 1.5d;
- final HostScore hostScore0 = new HostScore("H0", .2d/* rawScore */,
- totalRawScore, null/*scoringRule*/, null/* metrics */); // score=.533
- final HostScore hostScore1 = new HostScore("H1", .5d/* rawScore */,
- totalRawScore, null/*scoringRule*/, null/* metrics */); // score=.333, total=.866
- final HostScore hostScore2 = new HostScore("H2", .8d/* rawScore */,
- totalRawScore, null/*scoringRule*/, null/* metrics */); // score=.133, total=1.00
+//
+// /*
+// * The load scores for those hosts.
+// *
+// * Note: These need to be ordered by increasing load. That is how they
+// * are sorted by the LBS code. The running totals in terms of the
+// * normalized load.
+// */
+// final double totalRawScore = 1.5d;
+// final HostScore hostScore0 = new HostScore("H0", .2d); // score=.533
+// final HostScore hostScore1 = new HostScore("H1", .5d); // score=.333, total=.866
+// final HostScore hostScore2 = new HostScore("H2", .8d); // score=.133, total=1.00
+//
+// /*
+// * Show scores. Decision is made on the *normalized* score. Since we
+// * want to assign work based on inverse load, "normalized" is defined as
+// * (1-load)/total
+// */
+// log.warn("H1=" + hostScore0);
+// log.warn("H2=" + hostScore1);
+// log.warn("H3=" + hostScore2);
+//
+// // verify total of the raw scores.
+// assertEquals(totalRawScore, hostScore0.getRawScore() + hostScore1.getRawScore()
+// + hostScore2.getRawScore());
+//
+// // verify individual normalized scores.
+// assertEquals(hostScore0.getScore(), (1d-hostScore0.getRawScore()) / totalRawScore);
+// assertEquals(hostScore1.getScore(), (1d-hostScore1.getRawScore()) / totalRawScore);
+// assertEquals(hostScore2.getScore(), (1d-hostScore2.getRawScore()) / totalRawScore);
+//
+// // verify total of the normalized scores.
+// assertEquals(1d, hostScore0.getScore() + hostScore1.getScore() + hostScore2.getScore());
+//
+// // Arrange host scores in some perturbed order.
+// final HostScore[] hostScores = new HostScore[] { hostScore2,
+// hostScore0, hostScore1 };
+//
+// // Sort by increasing normalized score (decreasing free capacity).
+// Arrays.sort(hostScores);
+//
+// log.warn("hostScores[0]=" + hostScores[0]);
+// log.warn("hostScores[1]=" + hostScores[1]);
+// log.warn("hostScores[2]=" + hostScores[2]);
+//
+// // verify imposed ordering on (1-load).
+// assertSameRef(hostScores[2], hostScore0); // most load (1-load = .533)
+// assertSameRef(hostScores[1], hostScore1); // middle load (1-load = .333)
+// assertSameRef(hostScores[0], hostScore2); // least load (1-load = .133)
+//
+// // Verify ordering (increasing normalized score).
+// assertTrue(hostScores[0].getScore() < hostScores[1].getScore());
+// assertTrue(hostScores[1].getScore() < hostScores[2].getScore());
+//
+// final double threshold0 = 0d;
+// final double threshold1 = hostScores[0].getScore();
+// final double threshold2 = threshold1 + hostScores[1].getScore();
+// log.warn("threshold0=" + threshold0);
+// log.warn("threshold1=" + threshold1);
+// log.warn("threshold2=" + threshold2);
+//
+// {
+// final HostScore actualHost = AbstractHostLBSPolicy.getHost(
+// threshold0, hostScores);
+//
+// assertSameRef(hostScores[0], actualHost);
+// }
+//
+// {
+// final HostScore actualHost = AbstractHostLBSPolicy.getHost(
+// (threshold1 - .001), hostScores);
+//
+// assertSameRef(hostScores[0], actualHost);
+// }
+//
+// {
+// final HostScore actualHost = AbstractHostLBSPolicy.getHost(
+// threshold1, hostScores);
+//
+// assertSameRef(hostScores[1], actualHost);
+// }
+//
+// {
+// final HostScore actualHost = AbstractHostLBSPolicy.getHost(
+// (threshold2 - .001), hostScores);
+//
+// assertSameRef(hostScores[1], actualHost);
+// }
+//
+// {
+// final HostScore actualHost = AbstractHostLBSPolicy.getHost(
+// threshold2, hostScores);
+//
+// assertSameRef(hostScores[2], actualHost);
+// }
+//
+// {
+// final HostScore actualHost = AbstractHostLBSPolicy.getHost(
+// 1d - .0001, hostScores);
+//
+// assertSameRef(hostScores[2], actualHost);
+// }
+//
+// {
+// try {
+// AbstractHostLBSPolicy.getHost(1d, hostScores);
+// } catch (IllegalArgumentException ex) {
+// // ignore.
+// if (log.isInfoEnabled())
+// log.info(ex);
+// }
+// }
+//
+// {
+// try {
+// AbstractHostLBSPolicy.getHost(-.00001d, hostScores);
+// } catch (IllegalArgumentException ex) {
+// // ignore.
+// if (log.isInfoEnabled())
+// log.info(ex);
+// }
+// }
+//
+// // The services.
+// final UUID A = UUID.randomUUID();
+// final UUID B = UUID.randomUUID();
+// final UUID C = UUID.randomUUID();
+//
+// final ServiceScore serviceA = new ServiceScore(A, hostScore0.getHostname(), toRequestURI(hostScore0.getHostname()));
+// final ServiceScore serviceB = new ServiceScore(B, hostScore1.getHostname(), toRequestURI(hostScore1.getHostname()));
+// final ServiceScore serviceC = new ServiceScore(C, hostScore2.getHostname(), toRequestURI(hostScore2.getHostname()));
+//
+// /*
+// * Now check the method to identify a specific joined service known to
+// * be running on that host.
+// */
+// {
+// {
+//
+// // Run with a known seed.
+// final Random rand = new Random(1L);
+//
+// final ServiceScore[] serviceScores = new ServiceScore[] { //
+// serviceA //
+// };
+//
+// final ServiceScore actualService = AbstractHostLBSPolicy
+// .getService(rand, hostScore0, serviceScores);
+//
+// assertTrue(actualService == serviceA);
+//
+// }
+//
+// {
+//
+// // Run with a known seed.
+// final Random rand = new Random(1L);
+//
+// final ServiceScore[] serviceScores = new ServiceScore[] { //
+// serviceA, serviceB, serviceC
+// };
+//
+// final ServiceScore actualService = AbstractHostLBSPolicy
+// .getService(rand, hostScore0, serviceScores);
+//
+// assertTrue(actualService == serviceA);
+//
+// }
+//
+// }
- /*
- * Show scores. Decision is made on the *normalized* score. Since we
- * want to assign work based on inverse load, "normalized" is defined as
- * (1-load)/total
- */
- log.warn("H1=" + hostScore0);
- log.warn("H2=" + hostScore1);
- log.warn("H3=" + hostScore2);
-
- // verify total of the raw scores.
- assertEquals(totalRawScore, hostScore0.getRawScore() + hostScore1.getRawScore()
- + hostScore2.getRawScore());
-
- // verify individual normalized scores.
- assertEquals(hostScore0.getScore(), (1d-hostScore0.getRawScore()) / totalRawScore);
- assertEquals(hostScore1.getScore(), (1d-hostScore1.getRawScore()) / totalRawScore);
- assertEquals(hostScore2.getScore(), (1d-hostScore2.getRawScore()) / totalRawScore);
-
- // verify total of the normalized scores.
- assertEquals(1d, hostScore0.getScore() + hostScore1.getScore() + hostScore2.getScore());
-
- // Arrange host scores in some perturbed order.
- final HostScore[] hostScores = new HostScore[] { hostScore2,
- hostScore0, hostScore1 };
-
- // Sort by increasing normalized score (decreasing free capacity).
- Arrays.sort(hostScores);
-
- log.warn("hostScores[0]=" + hostScores[0]);
- log.warn("hostScores[1]=" + hostScores[1]);
- log.warn("hostScores[2]=" + hostScores[2]);
-
- // verify imposed ordering on (1-load).
- assertSameRef(hostScores[2], hostScore0); // most load (1-load = .533)
- assertSameRef(hostScores[1], hostScore1); // middle load (1-load = .333)
- assertSameRef(hostScores[0], hostScore2); // least load (1-load = .133)
-
- // Verify ordering (increasing normalized score).
- assertTrue(hostScores[0].getScore() < hostScores[1].getScore());
- assertTrue(hostScores[1].getScore() < hostScores[2].getScore());
-
- final double threshold0 = 0d;
- final double threshold1 = hostScores[0].getScore();
- final double threshold2 = threshold1 + hostScores[1].getScore();
- log.warn("threshold0=" + threshold0);
- log.warn("threshold1=" + threshold1);
- log.warn("threshold2=" + threshold2);
-
- {
- final HostScore actualHost = AbstractHostLBSPolicy.getHost(
- threshold0, hostScores);
-
- assertSameRef(hostScores[0], actualHost);
- }
-
- {
- final HostScore actualHost = AbstractHostLBSPolicy.getHost(
- (threshold1 - .001), hostScores);
-
- assertSameRef(hostScores[0], actualHost);
- }
-
- {
- final HostScore actualHost = AbstractHostLBSPolicy.getHost(
- threshold1, hostScores);
-
- assertSameRef(hostScores[1], actualHost);
- }
-
- {
- final HostScore actualHost = AbstractHostLBSPolicy.getHost(
- (threshold2 - .001), hostScores);
-
- assertSameRef(hostScores[1], actualHost);
- }
-
- {
- final HostScore actualHost = AbstractHostLBSPolicy.getHost(
- threshold2, hostScores);
-
- assertSameRef(hostScores[2], actualHost);
- }
-
- {
- final HostScore actualHost = AbstractHostLBSPolicy.getHost(
- 1d - .0001, hostScores);
-
- assertSameRef(hostScores[2], actualHost);
- }
-
- {
- try {
- AbstractHostLBSPolicy.getHost(1d, hostScores);
- } catch (IllegalArgumentException ex) {
- // ignore.
- if (log.isInfoEnabled())
- log.info(ex);
- }
- }
-
- {
- try {
- AbstractHostLBSPolicy.getHost(-.00001d, hostScores);
- } catch (IllegalArgumentException ex) {
- // ignore.
- if (log.isInfoEnabled())
- log.info(ex);
- }
- }
-
- // The services.
- final UUID A = UUID.randomUUID();
- final UUID B = UUID.randomUUID();
- final UUID C = UUID.randomUUID();
-
- final ServiceScore serviceA = new ServiceScore(A, hostScore0.getHostname(), toRequestURI(hostScore0.getHostname()));
- final ServiceScore serviceB = new ServiceScore(B, hostScore1.getHostname(), toRequestURI(hostScore1.getHostname()));
- final ServiceScore serviceC = new ServiceScore(C, hostScore2.getHostname(), toRequestURI(hostScore2.getHostname()));
-
- /*
- * Now check the method to identify a specific joined service known to
- * be running on that host.
- */
- {
- {
-
- // Run with a known seed.
- final Random rand = new Random(1L);
-
- final ServiceScore[] serviceScores = new ServiceScore[] { //
- serviceA //
- };
-
- final ServiceScore actualService = AbstractHostLBSPolicy
- .getService(rand, hostScore0, serviceScores);
-
- assertTrue(actualService == serviceA);
-
- }
-
- {
-
- // Run with a known seed.
- final Random rand = new Random(1L);
-
- final ServiceScore[] serviceScores = new ServiceScore[] { //
- serviceA, serviceB, serviceC
- };
-
- final ServiceScore actualService = AbstractHostLBSPolicy
- .getService(rand, hostScore0, serviceScores);
-
- assertTrue(actualService == serviceA);
-
- }
-
- }
-
}
/**
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 14:16:22
|
Revision: 8311
http://sourceforge.net/p/bigdata/code/8311
Author: thompsonbry
Date: 2014-05-14 14:16:19 +0000 (Wed, 14 May 2014)
Log Message:
-----------
bug fix to host selection logic.
See #624 (HA LBS)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 13:58:26 UTC (rev 8310)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 14:16:19 UTC (rev 8311)
@@ -784,9 +784,11 @@
for (HostScore tmp : hostScores) {
hostScore = tmp;
sum += (1d - hostScore.getScore());
- if (sum >= d) // scan further.
+ if (sum >= d) {
+ // found desired host.
break;
- break;
+ }
+ // scan further.
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-05-14 13:58:30
|
Revision: 8310
http://sourceforge.net/p/bigdata/code/8310
Author: thompsonbry
Date: 2014-05-14 13:58:26 +0000 (Wed, 14 May 2014)
Log Message:
-----------
Modified the HA load balancer:
- Added additional normalization logic.
- Added counters for tracking the #of local forwards, the #of proxies, and the #of errors.
- Increased the information rendered on the /status page for the HA load balancer.
Note: I have not yet fixed up the unit tests for the normalized scoring policy. I expect 2 failures for those tests.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HAStatusServletUtil.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractLBSPolicy.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostTable.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/IHALoadBalancerPolicy.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/ServiceScore.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/policy/NOPLBSPolicy.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-05-14 13:56:29 UTC (rev 8309)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-05-14 13:58:26 UTC (rev 8310)
@@ -38,6 +38,7 @@
import org.eclipse.jetty.proxy.ProxyServlet;
import com.bigdata.BigdataStatics;
+import com.bigdata.counters.CAT;
import com.bigdata.journal.AbstractJournal;
import com.bigdata.journal.IIndexManager;
import com.bigdata.journal.jini.ha.HAJournal;
@@ -133,6 +134,38 @@
}
+ /*
+ * Static declarations of some common exceptions to reduce overhead
+ * associated with filling in the stack traces.
+ */
+
+ /**
+ * There is no {@link IHALoadBalancerPolicy} currently declared.
+ */
+ private static final RuntimeException CAUSE_NO_LBS_POLICY = new RuntimeException(
+ "No LBS policy");
+
+ /**
+ * The <code>Request-URI</code> does not map to either {@link #PATH_LEADER}
+ * or {@link #PATH_READ}.
+ */
+ private static final RuntimeException CAUSE_BAD_REQUEST_URI = new RuntimeException(
+ "Bad Request-URI");
+
+ /**
+ * There is no {@link IHARequestURIRewriter} currently declared.
+ */
+ private static final RuntimeException CAUSE_NO_REWRITER_POLICY = new RuntimeException(
+ "No rewriter policy");
+
+ /**
+ * The destination could not be validated.
+ *
+ * @see #validateDestination(String, int)
+ */
+ private static final RuntimeException CAUSE_DESTINATION_NOT_VALID = new RuntimeException(
+ "Could not validate destination");
+
public HALoadBalancerServlet() {
super();
@@ -192,6 +225,24 @@
private final AtomicReference<IHARequestURIRewriter> rewriterRef = new AtomicReference<IHARequestURIRewriter>();
/**
+ * The number of requests that were forwarded to the local service.
+ */
+ private final CAT nforward = new CAT();
+
+ /**
+ * The number of requests that were proxied some service.
+ */
+ private final CAT nproxy = new CAT();
+
+ /**
+ * The number of requests for which {@link #rewriteURI(HttpServletRequest)}
+ * trapped an error.
+ * <p>
+ * Note: Such requests wind up being forwarded to the local service.
+ */
+ private final CAT nerror = new CAT();
+
+ /**
* Change the {@link IHALoadBalancerPolicy} associated with this instance of
* this servlet. The new policy will be installed iff it can be initialized
* successfully. The old policy will be destroyed iff the new policy is
@@ -524,30 +575,44 @@
* @return The {@link IHALoadBalancerPolicy}[] -or- <code>null</code> if
* there are no {@link HALoadBalancerServlet}s.
*/
- public static IHALoadBalancerPolicy[] getLBSPolicy(
- final ServletContext servletContext) {
+ public static String toString(final ServletContext servletContext) {
final HALoadBalancerServlet[] servlets = getServlets(servletContext);
if (servlets == null || servlets.length == 0) {
// None running.
- return null;
+ return "Not running";
}
- final IHALoadBalancerPolicy[] a = new IHALoadBalancerPolicy[servlets.length];
-
+ final StringBuilder sb = new StringBuilder();
+
for (int i = 0; i < servlets.length; i++) {
- a[i] = servlets[i].getLBSPolicy();
+ sb.append(servlets[i].toString());
}
+
+ return sb.toString();
+
+ }
- return a;
+ @Override
+ public String toString() {
+ return super.toString()//
+ + "{prefix=" + prefix//
+ + ",policy=" + policyRef.get()//
+ + ",rewriter=" + rewriterRef.get()//
+ + ",nforward=" + nforward.estimate_get()//
+ + ",nproxy=" + nproxy.estimate_get()//
+ + ",nerror=" + nerror.estimate_get()//
+ + "}"//
+ ;
+
}
-
+
/**
* {@inheritDoc}
* <p>
@@ -714,7 +779,7 @@
* LBS is disabled. Strip LBS prefix from the requestURI and forward
* the request to servlet on this host (NOP LBS).
*/
- forwardToThisService(isLeaderRequest,request, response);
+ forwardToLocalService(isLeaderRequest,request, response);
return;
}
@@ -730,7 +795,7 @@
* this service is not the leader. Look at it again under a debugger
* and optimize the code paths.
*/
- if (policy.service(isLeaderRequest, request, response)) {
+ if (policy.service(isLeaderRequest, this, request, response)) {
// Return immediately if the response was committed.
return;
@@ -760,7 +825,7 @@
* @throws IOException
* @throws ServletException
*/
- static public void forwardToThisService(//
+ public void forwardToLocalService(//
final boolean isLeaderRequest,//
final HttpServletRequest request, //
final HttpServletResponse response//
@@ -817,15 +882,17 @@
* stripped off the prefix for the LBS.
*/
- if (log.isInfoEnabled())
- log.info("forward: " + path + " => " + newPath);
-
// Get dispatched for the new requestURL path.
final RequestDispatcher requestDispatcher = request
.getRequestDispatcher(newPath);
try {
+
+ nforward.increment();
+ if (log.isInfoEnabled())
+ log.info("forward: " + path + " => " + newPath);
+
// forward to a local servlet.
requestDispatcher.forward(request, response);
@@ -848,76 +915,152 @@
*
* @return The full prefix.
*
- * TODO This may need to be configurable. It is currently static
- * since
- * {@link #forwardToThisService(boolean, HttpServletRequest, HttpServletResponse)}
- * is static.
+ * TODO This may need to be configurable.
*/
- private static String getFullPrefix(final boolean isLeaderRequest,
+ private String getFullPrefix(final boolean isLeaderRequest,
final String prefix) {
- final String full_prefix = isLeaderRequest ? prefix + "/leader"
- : prefix + "/read";
+ final String full_prefix = isLeaderRequest ? prefix + PATH_LEADER
+ : prefix + PATH_READ;
return full_prefix;
}
/**
+ * Wrapper invokes {@link #doRewriteURI(HttpServletRequest)} and handles
+ * any thrown exceptions.
+ *
+ * @see #doRewriteURI(HttpServletRequest)
+ */
+ @Override
+ final protected URI rewriteURI(final HttpServletRequest request) {
+
+ try {
+
+ final URI rewritten = doRewriteURI(request);
+
+ if (rewritten != null) {
+
+ // Track #of requests that are proxied.
+ nproxy.increment();
+
+ }
+
+ return rewritten;
+
+ } catch (Throwable t) {
+
+ /*
+ * Could not rewrite.
+ */
+
+ if (InnerCause.isInnerCause(t, InterruptedException.class)) {
+
+ throw new RuntimeException(t);
+
+ }
+
+ nerror.increment();
+
+ if (log.isDebugEnabled()) {
+ // full stack trace.
+ log.warn(t, t);
+ } else {
+ // just the message.
+ log.warn(t);
+ }
+
+ /*
+ * Could not rewrite.
+ *
+ * Return [null]. This will cause the onRewriteFailed() to be
+ * invoked. That will do a local forward. If the request can not be
+ * handled locally, then the local service will generate an error
+ * message.
+ *
+ * Note: This pattern means that we do not throw errors arising from
+ * the decision to rewrite the request. They are logger (above) and
+ * then the local forwarding logic is applied. This can mask some
+ * errors since they will only appear in the log. However, this does
+ * make the rewrite logic more robust.
+ */
+
+ return null;
+
+ }
+
+ }
+
+ /**
+ * Hook allows the servlet to rewrite the request.
+ *
* For update requests, rewrite the requestURL to the service that is the
* quorum leader. For read requests, rewrite the requestURL to the service
* having the least load.
+ *
+ * @param request
+ * The request.
+ *
+ * @return Return the {@link URI} if the request should be proxied to
+ * another service -or- <code>null</code> if the request should be
+ * locally forwarded.
*/
- @Override
- protected URI rewriteURI(final HttpServletRequest request) {
-
+ protected URI doRewriteURI(final HttpServletRequest request) {
+
final IHALoadBalancerPolicy policy = policyRef.get();
if (policy == null) {
- // Could not rewrite.
- return null;
+ // No policy. Can not rewrite.
+ throw CAUSE_NO_LBS_POLICY;
}
final String originalRequestURI = request.getRequestURI();
- if (!originalRequestURI.startsWith(prefix))
- return null;
+ if (!originalRequestURI.startsWith(prefix)) {
+ // Request is not for this servlet.
+ throw CAUSE_BAD_REQUEST_URI;
+ }
final Boolean isLeaderRequest = isLeaderRequest(request);
if (isLeaderRequest == null) {
// Neither /LBS/leader -nor- /LBS/read.
- return null;
+ throw CAUSE_BAD_REQUEST_URI;
}
final String proxyToRequestURI;
-
- if(isLeaderRequest) {
- // Proxy to leader.
- proxyToRequestURI = policy.getLeaderURI(request);
- } else {
- // Proxy to any joined service.
- proxyToRequestURI = policy.getReaderURI(request);
- }
-
- if (log.isDebugEnabled())
- log.debug("proxyToRequestURI=" + proxyToRequestURI);
+ {
- if (proxyToRequestURI == null) {
- // Could not rewrite.
- return null;
+ if (isLeaderRequest) {
+ // Proxy to leader.
+ proxyToRequestURI = policy.getLeaderURI(request);
+ } else {
+ // Proxy to any joined service.
+ proxyToRequestURI = policy.getReaderURI(request);
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("proxyToRequestURI=" + proxyToRequestURI);
+
+ if (proxyToRequestURI == null) {
+ /*
+ * The LBS policy made a choice not to rewrite this request (not
+ * an error, but a deliberate choice).
+ */
+ return null;
+ }
+
}
- // the full LBS prefix (includes /leader or /read).
+ // The full LBS prefix (includes /leader or /read).
final String full_prefix = getFullPrefix(isLeaderRequest, prefix);
// The configured Request-URL rewriter.
final IHARequestURIRewriter rewriter = rewriterRef.get();
if (rewriter == null) {
- // Could not rewrite.
- log.warn("No rewriter: requestURI=" + originalRequestURI);
- return null;
+ throw CAUSE_NO_REWRITER_POLICY;
}
// Re-write requestURL.
@@ -932,13 +1075,15 @@
// Normalize the request.
final URI rewrittenURI = URI.create(uri.toString()).normalize();
- if (!validateDestination(rewrittenURI.getHost(), rewrittenURI.getPort()))
- return null;
+ if (!validateDestination(rewrittenURI.getHost(), rewrittenURI.getPort())) {
+ throw CAUSE_DESTINATION_NOT_VALID;
+ }
if (log.isInfoEnabled())
log.info("rewrote: " + originalRequestURI + " => " + rewrittenURI);
return rewrittenURI;
+
}
/**
@@ -969,7 +1114,7 @@
}
// Forward to this service (let it generate an error message).
- forwardToThisService(isLeaderRequest, request, response);
+ forwardToLocalService(isLeaderRequest, request, response);
// response.sendError(HttpServletResponse.SC_FORBIDDEN);
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HAStatusServletUtil.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HAStatusServletUtil.java 2014-05-14 13:56:29 UTC (rev 8309)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HAStatusServletUtil.java 2014-05-14 13:58:26 UTC (rev 8310)
@@ -30,7 +30,6 @@
import java.security.DigestException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
import java.util.Iterator;
import java.util.UUID;
import java.util.concurrent.TimeoutException;
@@ -65,7 +64,6 @@
import com.bigdata.quorum.zk.ZKQuorumClient;
import com.bigdata.quorum.zk.ZKQuorumImpl;
import com.bigdata.rdf.sail.webapp.StatusServlet.DigestEnum;
-import com.bigdata.rdf.sail.webapp.lbs.IHALoadBalancerPolicy;
import com.bigdata.zookeeper.DumpZookeeper;
/**
@@ -290,15 +288,12 @@
+ journal.getSnapshotManager().getRestorePolicy())
.node("br").close();
- // LBS policy
+ // HA Load Balancer.
{
- final IHALoadBalancerPolicy[] a = HALoadBalancerServlet
- .getLBSPolicy(req.getServletContext());
-
p.text("Service: LBSPolicy="
- + (a == null ? "N/A" : Arrays.toString(a)))
- .node("br").close();
+ + HALoadBalancerServlet.toString(req
+ .getServletContext())).node("br").close();
}
// if(true) {
// /*
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 13:56:29 UTC (rev 8309)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractHostLBSPolicy.java 2014-05-14 13:58:26 UTC (rev 8310)
@@ -23,7 +23,6 @@
package com.bigdata.rdf.sail.webapp.lbs;
import java.io.IOException;
-import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -47,6 +46,7 @@
import com.bigdata.journal.Journal;
import com.bigdata.journal.PlatformStatsPlugIn;
import com.bigdata.journal.jini.ha.HAJournal;
+import com.bigdata.quorum.Quorum;
import com.bigdata.rdf.sail.webapp.HALoadBalancerServlet;
import com.bigdata.util.InnerCause;
@@ -127,10 +127,42 @@
}
+ /*
+ * Static declarations of some common exceptions to reduce overhead
+ * associated with filling in the stack traces.
+ */
+
/**
+ * The {@link HostTable} is empty (no hosts).
+ */
+ private static final RuntimeException CAUSE_EMPTY_HOST_TABLE = new RuntimeException(
+ "Empty host table.");
+
+ /**
+ * The service table is empty (no services).
+ */
+ private static final RuntimeException CAUSE_EMPTY_SERVICE_TABLE = new RuntimeException(
+ "Empty service table.");
+
+ /**
+ * The load balancing logic failed to select a host to handle the
+ * request.
+ */
+ private static final RuntimeException CAUSE_NO_HOST_SELECTED = new RuntimeException(
+ "No host selected for request.");
+
+ /**
+ * The load balancing logic failed to select a service to handle the
+ * request.
+ */
+ private static final RuntimeException CAUSE_NO_SERVICE_SELECTED = new RuntimeException(
+ "No service selected for request.");
+
+ /**
* @see InitParams#LOCAL_FORWARD_THRESHOLD
*/
private final AtomicReference<Double> localForwardThresholdRef = new AtomicReference<Double>();
+
/**
* The rule used to score the {@link IHostMetrics}.
*
@@ -465,6 +497,9 @@
final HostTable newHostTable = normalizeHostScores(scoringRule,
hostMetricsMap);
+ if (log.isTraceEnabled())
+ log.trace("newHostTable=" + newHostTable);
+
// Set the host table.
hostTableRef.set(newHostTable);
@@ -472,13 +507,18 @@
/**
* Compute and return the normalized load across the known hosts.
+ * <p>
+ * Note: This needs to be done only for those hosts that are associated with
+ * the {@link Quorum} members. If we do it for the other hosts then the
+ * normalization is not meaningful since we will only load balance across
+ * the services that are joined with a met {@link Quorum}.
*
* @param scoringRule
* The {@link IHostScoringRule} used to integrate the per-host
* performance metrics.
* @param hostMetricsMap
* The per-host performance metrics for the known hosts.
- *
+ *
* @return The normalized host workload.
*/
private static HostTable normalizeHostScores(
@@ -505,24 +545,39 @@
{
+ /*
+ * TODO Since the scoring rule does not produce normalized host
+ * scores, we do not know how the NO_INFO will be ordered with
+ * respect to those hosts for which the scoring rule was
+ * successfully applied. This could be made to work either by
+ * flagging hosts without metrics or by pushing down the handling of
+ * a [null] metrics reference into the scoring rule, which would
+ * know how to return a "median" value.
+ */
+ final double NO_INFO = .5d;
+
int i = 0;
for (Map.Entry<String, IHostMetrics> e : hostMetricsMap.entrySet()) {
final String hostname = e.getKey();
+ assert hostname != null; // Note: map keys are never null.
+
final IHostMetrics metrics = e.getValue();
if (log.isDebugEnabled())
log.debug("hostname=" + hostname + ", metrics=" + metrics);
- double hostScore = scoringRule.getScore(metrics);
+ // flag host if no load information is available.
+ double hostScore = metrics == null ? NO_INFO : scoringRule
+ .getScore(metrics);
if (hostScore < 0) {
log.error("Negative score: " + hostname);
- hostScore = 0d;
+ hostScore = NO_INFO;
}
@@ -530,21 +585,29 @@
hostScores[i] = hostScore;
+ metrics2[i] = metrics;
+
totalScore += hostScore;
i++;
}
+ if (totalScore == 0) {
+
+ /*
+ * If totalScore is zero, then weight all hosts equally as
+ * (1/nhosts).
+ */
+
+ totalScore = nhosts;
+
+ }
+
}
/*
* Normalize the per-hosts scores.
- *
- * Note: This needs to be done only for those hosts that are associated
- * with the quorum members. If we do it for the other hosts then the
- * normalization is not meaningful. That is why we first filter for only
- * those hosts that are running services associated with this quorum.
*/
HostScore thisHostScore = null;
@@ -555,28 +618,13 @@
final String hostname = hostnames[i];
- final HostScore hostScore;
- if (totalScore == 0d) {
+ // Normalize host scores.
+ final HostScore hostScore = scores[i] = new HostScore(hostname,
+ hostScores[i], totalScore, metrics2[i], scoringRule);
- /*
- * If totalScore is zero, then weight all hosts equally as
- * (1/nhosts).
- */
- hostScore = new HostScore(hostname, 1d, nhosts,
- scoringRule, metrics2[i]);
+ if (thisHostScore != null && hostScore.isThisHost()) {
- } else {
-
- // Normalize host scores.
- hostScore = new HostScore(hostname, hostScores[i],
- totalScore, scoringRule, metrics2[i]);
-
- }
-
- scores[i] = hostScore;
-
- if (hostScore.isThisHost()) {
-
+ // The first score discovered for this host.
thisHostScore = hostScore;
}
@@ -585,9 +633,6 @@
}
- // sort into ascending order (increasing activity).
- Arrays.sort(scores);
-
// for (int i = 0; i < scores.length; i++) {
//
// scores[i].rank = i;
@@ -596,21 +641,24 @@
//
// }
- if (scores.length > 0) {
+// // Sort into order by decreasing load.
+// Arrays.sort(scores);
- if (log.isDebugEnabled()) {
+// if (scores.length > 0) {
+//
+// if (log.isDebugEnabled()) {
+//
+// log.debug("The most active index was: "
+// + scores[scores.length - 1]);
+//
+// log.debug("The least active index was: " + scores[0]);
+//
+// log.debug("This host: " + thisHostScore);
+//
+// }
+//
+// }
- log.debug("The most active index was: "
- + scores[scores.length - 1]);
-
- log.debug("The least active index was: " + scores[0]);
-
- log.debug("This host: " + thisHostScore);
-
- }
-
- }
-
return new HostTable(thisHostScore, scores);
}
@@ -626,32 +674,41 @@
final ServiceScore[] serviceScores = serviceTableRef.get();
if (hostScores == null || hostScores.length == 0) {
-
// Can't do anything.
- log.warn("Empty host table.");
- return null;
-
+ throw CAUSE_EMPTY_HOST_TABLE;
}
if (serviceScores == null) {
+ // No services.
+ throw CAUSE_EMPTY_SERVICE_TABLE;
+ }
- // No services. Can't proxy.
- log.warn("No service scores.");
- return null;
+ final HostScore hostScore = getHost(rand.nextDouble(), hostScores);
+ if (hostScore == null) {
+ // None found.
+ throw CAUSE_NO_HOST_SELECTED;
}
- final HostScore hostScore = getHost(rand.nextDouble(), hostScores);
-
final ServiceScore serviceScore = getService(rand, hostScore,
serviceScores);
- if(serviceScore == null) {
+ if (serviceScore == null) {
// None found.
- return null;
+ throw CAUSE_NO_SERVICE_SELECTED;
}
-
- if(serviceScore.getServiceUUID().equals(serviceIDRef.get())) {
+
+ /*
+ * Track #of requests to each service.
+ *
+ * Note: ServiceScore.nrequests is incremented before we make the
+ * decision to do a local forward when the target is *this* host. This
+ * means that the /status page will still show the effect of the load
+ * balancer for local forwards. This is a deliberate decision.
+ */
+ serviceScore.nrequests.increment();
+
+ if (serviceScore.getServiceUUID().equals(serviceIDRef.get())) {
/*
* The target is *this* service. As an optimization, we return
* [null] so that the caller will perform a local forward (as part
@@ -720,17 +777,15 @@
/*
* Multiple hosts.
*
- * Note: Choice is inversely proportional to normalized
- * workload.
+ * Note: Choice is inversely proportional to normalized workload
+ * (1 - load).
*/
double sum = 0d;
- for (int i = 0; i < hostScores.length; i++) {
- hostScore = hostScores[i];
- sum += hostScore.getScore(); // score:=freeCapacity
- if (hostScore.getHostname() == null) // can't use w/o hostname.
- continue;
- if (d >= sum) // scan further.
- continue;
+ for (HostScore tmp : hostScores) {
+ hostScore = tmp;
+ sum += (1d - hostScore.getScore());
+ if (sum >= d) // scan further.
+ break;
break;
}
}
@@ -812,16 +867,6 @@
final ServiceScore serviceScore = foundServices.get(n);
- /*
- * Track #of requests to each service.
- *
- * Note: ServiceScore.nrequests is incremented before we make the
- * decision to do a local forward when the target is *this* host. This
- * means that the /status page will still show the effect of the load
- * balancer for local forwards. This is a deliberate decision.
- */
- serviceScore.nrequests.increment();
-
return serviceScore;
}
@@ -835,8 +880,10 @@
*/
@Override
protected boolean conditionallyForwardReadRequest(
- final HttpServletRequest request, final HttpServletResponse response)
- throws IOException {
+ final HALoadBalancerServlet servlet,//
+ final HttpServletRequest request, //
+ final HttpServletResponse response//
+ ) throws IOException {
final HostTable hostTable = hostTableRef.get();
@@ -846,8 +893,8 @@
if (thisHostScore != null
&& thisHostScore.getScore() <= localForwardThresholdRef.get()) {
- HALoadBalancerServlet.forwardToThisService(
- false/* isLeaderRequest */, request, response);
+ servlet.forwardToLocalService(false/* isLeaderRequest */, request,
+ response);
// request was handled.
return true;
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractLBSPolicy.java 2014-05-14 13:56:29 UTC (rev 8309)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/AbstractLBSPolicy.java 2014-05-14 13:58:26 UTC (rev 8310)
@@ -197,9 +197,12 @@
}
@Override
- public boolean service(final boolean isLeaderRequest,
- final HttpServletRequest request, final HttpServletResponse response)
- throws ServletException, IOException {
+ public boolean service(//
+ final boolean isLeaderRequest,
+ final HALoadBalancerServlet servlet,//
+ final HttpServletRequest request, //
+ final HttpServletResponse response//
+ ) throws ServletException, IOException {
/*
* Figure out whether the quorum is met and if this is the quorum
@@ -239,8 +242,7 @@
*
* @see #forwardToThisService()
*/
- HALoadBalancerServlet.forwardToThisService(isLeaderRequest,
- request, response);
+ servlet.forwardToLocalService(isLeaderRequest, request, response);
// request was handled.
return true;
@@ -258,7 +260,7 @@
* Provide an opportunity to forward a read request to the local
* service.
*/
- if (conditionallyForwardReadRequest(request, response)) {
+ if (conditionallyForwardReadRequest(servlet, request, response)) {
// Handled.
return true;
@@ -282,8 +284,10 @@
* @throws IOException
*/
protected boolean conditionallyForwardReadRequest(
- final HttpServletRequest request, final HttpServletResponse response)
- throws IOException {
+ final HALoadBalancerServlet servlet,
+ final HttpServletRequest request, //
+ final HttpServletResponse response//
+ ) throws IOException {
return false;
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java 2014-05-14 13:56:29 UTC (rev 8309)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostScore.java 2014-05-14 13:58:26 UTC (rev 8310)
@@ -25,8 +25,9 @@
import com.bigdata.counters.AbstractStatisticsCollector;
/**
- * Helper class assigns a raw and a normalized score to each host based on its
- * per-host.
+ * Helper class assigns a raw score (load) and a normalized score (normalized
+ * load) to each host based on its per-host metrics and a rule for computing the
+ * load of a host from those metrics.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
*/
@@ -48,13 +49,13 @@
private final double rawScore;
/**
- * The normalized score computed for that host.
+ * The normalized score for that host.
* <p>
* Note: The {@link #rawScore} is a measure of the <em>load</em> on a host.
- * The normalized {@link #score} is based on
- * <code>1-{@link #rawScore}</code> and is a measure of the normalized
- * amount of capacity to do more work on a host. Load balancing decision are
- * based on this normalized {@link #score}.
+ * The normalized {@link #score} is a measure of the normalized
+ * <em>load</em> on a host. Load balancing decision are based on this
+ * normalized {@link #score} (work is assigned to a host in inverse
+ * proportion to its normalized {@link #score}).
*/
private final double score;
@@ -78,23 +79,22 @@
// public double drank = -1d;
/**
- * Return the raw score for a host, which is a measure of the load on that
- * host. The measure is computed based on {@link IHostMetrics} using some
- * {@link IHostScoringRule}.
+ * Return the raw score (aka load) for a host. This raw score is an
+ * unnormalized measure of the load on that host. The measure is computed
+ * based on {@link IHostMetrics} using some {@link IHostScoringRule}.
*/
public double getRawScore() {
return rawScore;
}
/**
- * Return the measure of free capacity to do work on the host.
+ * Return the normalized load for the host.
* <p>
* Note: The {@link #getRawScore() rawScore} is a measure of the
- * <em>load</em> on a host. The normalized {@link #getScore() score} is
- * based on <code>1-{@link #getRawScore() rawScore}</code> and is a measure
- * of the normalized amount of capacity to do more work on a host. Load
- * balancing decision are based on this normalized {@link #getScore() score}
- * .
+ * <em>load</em> on a host. The normalized {@link #getScore() score} is the
+ * normalized load for a host. Load balancing decision are based on this
+ * normalized {@link #getScore() score} (work is assigned to hosts in
+ * inverse proportion to the normalized load of the host).
*/
public double getScore() {
return score;
@@ -106,10 +106,9 @@
}
/**
- * The {@link IHostMetrics} associated with this host (this is
- * informational. The {@link #getMetrics()} provide the detailed per-host
- * performance metrics that were intepreted by the {@link #getScoringRule()}
- * .
+ * The {@link IHostMetrics} associated with this host (optional). The
+ * {@link #getMetrics()} provide the detailed per-host performance metrics
+ * that were intepreted by the {@link #getScoringRule()} .
*/
public IHostMetrics getMetrics() {
return metrics;
@@ -117,7 +116,7 @@
/**
* The {@link IHostScoringRule} used to convert the {@link #getMetrics()}
- * into the {@link #getRawScore()}.
+ * into the {@link #getRawScore()} (optional).
*/
public IHostScoringRule getScoringRule() {
return scoringRule;
@@ -141,13 +140,32 @@
// + ", rank=" + rank //
// + ", drank=" + drank //
+ ", metrics=" + metrics //
+ + ", scoringRule=" + scoringRule //
+ "}";
}
- public HostScore(final String hostname, final double rawScore,
- final double totalRawScore, final IHostScoringRule scoringRule,
- final IHostMetrics metrics) {
+ /**
+ *
+ * @param hostname
+ * The hostname (required, must be non-empty).
+ * @param rawScore
+ * The unnormalized load for that host.
+ * @param totalRawScore
+ * The total unnormalized load across all hosts.
+ * @param metrics
+ * The performance metrics used to compute the unnormalized load
+ * for each host (optional).
+ * @param scoringRule
+ * The rule used to score those metrics (optional).
+ */
+ public HostScore(//
+ final String hostname,//
+ final double rawScore,//
+ final double totalRawScore, //
+ final IHostMetrics metrics,//
+ final IHostScoringRule scoringRule//
+ ) {
if (hostname == null)
throw new IllegalArgumentException();
@@ -190,25 +208,38 @@
}
- return (1d - rawScore) / totalRawScore;
+ final double score = rawScore / totalRawScore;
+ if (score < 0 || score > 1) {
+
+ throw new RuntimeException("score(" + score + ") := rawScore("
+ + rawScore + ") / totalRawScore(" + totalRawScore + ")");
+
+ }
+
+ return score;
+
}
/**
- * Places elements into order by increasing normalized {@link #getScore()}.
- * The {@link #getHostname()} is used to break any ties (but this does not
- * help when all services are on the same host).
+ * Places elements into order by decreasing {@link #getScore() normalized
+ * load}. The {@link #getHostname()} is used to break any ties (but this
+ * does not help when all services are on the same host).
+ * <p>
+ * Note: The ordering is not really material to anything. Stochastic load
+ * balancing decisions can be made without regard to this ordering using the
+ * {@link #getScore() normalized load}.
*/
@Override
public int compareTo(final HostScore arg0) {
if (score < arg0.score) {
- return -1;
+ return 1;
} else if (score > arg0.score) {
- return 1;
+ return -1;
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostTable.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostTable.java 2014-05-14 13:56:29 UTC (rev 8309)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/HostTable.java 2014-05-14 13:58:26 UTC (rev 8310)
@@ -24,7 +24,6 @@
import java.util.Arrays;
-
/**
* Class bundles together the set of {@link HostScore}s for services that are
* joined with the met quorum and the {@link HostScore} for this service (iff it
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/IHALoadBalancerPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/IHALoadBalancerPolicy.java 2014-05-14 13:56:29 UTC (rev 8309)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/IHALoadBalancerPolicy.java 2014-05-14 13:58:26 UTC (rev 8310)
@@ -50,6 +50,10 @@
* balanced over the joined services. UPDATEs MUST be handled by
* the leader. Read requests can be handled by any service that
* is joined with the met quorum.
+ * @param servlet
+ * The {@link HALoadBalancerServlet}. This is exposed in order to
+ * allow the {@link IHALoadBalancerPolicy} to perform a local
+ * forward of the request.
* @param request
* The request.
* @param response
@@ -57,9 +61,12 @@
*
* @return <code>true</code> iff the request was handled.
*/
- boolean service(final boolean isLeaderRequest,
- final HttpServletRequest request, final HttpServletResponse response)
- throws ServletException, IOException;
+ boolean service(//
+ final boolean isLeaderRequest,
+ final HALoadBalancerServlet servlet,//
+ final HttpServletRequest request, //
+ final HttpServletResponse response//
+ ) throws ServletException, IOException;
/**
* Return the Request-URI to which a non-idempotent request will be proxied.
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/ServiceScore.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/ServiceScore.java 2014-05-14 13:56:29 UTC (rev 8309)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/ServiceScore.java 2014-05-14 13:58:26 UTC (rev 8310)
@@ -34,12 +34,6 @@
/**
* Helper class caches metadata about an {@link HAGlue} service.
- * <p>
- * Note: This class is written fairly defensively. The fields can wind up being
- * left at their default values (typically <code>null</code>) if we are not able
- * to find the necessary information for the {@link HAGlue} service. Users of
- * this class must test for <code>null</code> values and skip over those
- * services since they have not been pre-resolved to a host and requestURL.
*/
public class ServiceScore {
@@ -131,6 +125,18 @@
}
+ /**
+ *
+ * @param serviceUUID
+ * The {@link UUID} for the service.
+ * @param hostname
+ * The hostname of the host on which the service is running.
+ * @param requestURI
+ * The Request-URI for the service.
+ *
+ * @throws IllegalArgumentException
+ * if any argument is <code>null</code>.
+ */
ServiceScore(final UUID serviceUUID, final String hostname,
final String requestURI) {
@@ -151,10 +157,38 @@
}
+ /**
+ * Factory for {@link ServiceScore} instances.
+ *
+ * @param indexManager
+ * The index manager (required).
+ * @param contextPath
+ * The Context-Path of the web application (/bigdata).
+ * @param serviceUUID
+ * The {@link UUID} of the service.
+ *
+ * @return The {@link ServiceScore}.
+ *
+ * @throws IllegalArgumentException
+ * if any argument is <code>null</code>.
+ * @throws ClassCastException
+ * if the {@link IIndexManager} is not an {@link HAJournal}.
+ * @throws IOException
+ * If an RMI to the {@link HAGlue} proxy fails.
+ * @throws RuntimeException
+ * if anything else goes wrong.
+ */
static public ServiceScore newInstance(final IIndexManager indexManager,
final String contextPath, final UUID serviceUUID)
- throws IOException, RuntimeException {
+ throws IllegalArgumentException, ClassCastException, IOException,
+ RuntimeException {
+ if (indexManager == null)
+ throw new IllegalArgumentException();
+
+ if (contextPath == null)
+ throw new IllegalArgumentException();
+
if (serviceUUID == null)
throw new IllegalArgumentException();
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/policy/NOPLBSPolicy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/policy/NOPLBSPolicy.java 2014-05-14 13:56:29 UTC (rev 8309)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/lbs/policy/NOPLBSPolicy.java 2014-05-14 13:58:26 UTC (rev 8310)
@@ -49,15 +49,17 @@
private static final long serialVersionUID = 1L;
@Override
- public boolean service(final boolean isLeaderRequest,
- final HttpServletRequest request, final HttpServletResponse response)
- throws IOException, ServletException {
+ public boolean service(//
+ final boolean isLeaderRequest,//
+ final HALoadBalancerServlet servlet,//
+ final HttpServletRequest request, //
+ final HttpServletResponse response//
+ ) throws IOException, ServletException {
if (!isLeaderRequest) {
// Always handle read requests locally.
- HALoadBalancerServlet.forwardToThisService(isLeaderRequest,
- request, response);
+ servlet.forwardToLocalService(isLeaderRequest, request, response);
// Request was handled.
return true;
@@ -65,7 +67,7 @@
}
// Proxy update requests to the quorum leader.
- return super.service(isLeaderRequest, request, response);
+ return super.service(isLeaderRequest, servlet, request, response);
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
4 messages has been excluded from this view by a project administrator.
