Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[8494] branches/BIGDATA_RELEASE_1_3_0
|
From: <tho...@us...> - 2014-06-17 13:28:15
|
Revision: 8494
http://sourceforge.net/p/bigdata/code/8494
Author: thompsonbry
Date: 2014-06-17 13:28:07 +0000 (Tue, 17 Jun 2014)
Log Message:
-----------
Fix for #965 (LBS does not work with HA1)
The root cause was the init() method on HALoadBalancerServlet. init() disabled the LBS unless the NSS was in an HA GT 1 deployment model. I have modified the code to always enable the LBS servlet. This allows it to correctly rewrite itself out of the request when in the HA1 or non-HA modes.
There are some code paths that need to be updated because either (a) they do not make an explicit choice about whether or not to use the LBS; (b) they do not parametrize the ContextPath of the web application; or (c) they are in the wrong package (client code should be in a separate patch from server code).
Changes are to:
- HALoadBalancerServlet: init() always succeeds. This fixes the core problem for this ticket.
- TestHA1JournalServer: the test is linked to this ticket.
- RemoteServiceCallImpl: Modified to use declared service configuration information to decide whether or not to use the LBS pattern for the remote service. The default is false, which works for all cases. The default may be overridden to be true if the end point is known to expose the bigdata LBS pattern.
- IServiceOptions: added the isBigdataLBS() method.
- ServiceOptionsBase: added default=false for isBigdataLBS() and setBigdataLBS() methods.
- BigdataSailFactory: Added "FIXME" - This does not support the HA load balancer pattern and does not parameterize the value of the ContextPath. Also, should this class be part of the "client" package?
- BigdataSailRemoteRepository: added constructor variant that accepts the boolean useLBS argument. The version of the constructor without that argument now defaults to useLBS:=true. This changes the default behavior of the client(!).
- RemoteRepository: deprecated the constructor version that does not accept the useLBS parameter. This version of the constructor still default to useLBS:=false. It tends to be used from some less common code paths.
- RemoteRepositoryManager: modified to specify useLBS:=true by default.
- TestNSSHealthCheck: code cleanup.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1JournalServer.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/INativeServiceOptions.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/IServiceOptions.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/RemoteServiceCallImpl.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceCallCreateParams.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceOptionsBase.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailFactory.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/remote/BigdataSailRemoteRepository.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepository.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepositoryManager.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/health/TestNSSHealthCheck.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1JournalServer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1JournalServer.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1JournalServer.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -123,17 +123,24 @@
}
+ /**
+ * A simple transaction test against an HA1 mode server using the LBS.
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/965" > Cannot run queries in
+ * LBS mode with HA1 setup </a>
+ */
public void testSimpleTransactionLBS() throws Exception {
-
- doStartA();
-
- serverA.awaitHAReady(2, TimeUnit.SECONDS);
-
- awaitCommitCounter(1, new HAGlue[] { serverA });
-
- simpleTransactionLBS();
-
- awaitCommitCounter(2, new HAGlue[] { serverA });
+
+ doStartA();
+
+ serverA.awaitHAReady(2, TimeUnit.SECONDS);
+
+ awaitCommitCounter(1, new HAGlue[] { serverA });
+
+ simpleTransactionLBS();
+
+ awaitCommitCounter(2, new HAGlue[] { serverA });
+
}
public void testMultiTransaction() throws Exception {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/INativeServiceOptions.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/INativeServiceOptions.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/INativeServiceOptions.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -32,7 +32,6 @@
* Additional options for native services.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public interface INativeServiceOptions extends IServiceOptions {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/IServiceOptions.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/IServiceOptions.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/IServiceOptions.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -37,7 +37,6 @@
* Options and metadata for service end points.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public interface IServiceOptions {
@@ -78,5 +77,13 @@
* query planner has locked in the join evaluation order.
*/
boolean isRunFirst();
+
+ /**
+ * Return <code>true</code> if the remote service is known to be a bigdata
+ * service that exposes the HA load balancer servlet (default
+ * <code>false</code>). The default may be overridden iff the end point is
+ * known to expose the bigdata LBS pattern.
+ */
+ boolean isBigdataLBS();
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/RemoteServiceCallImpl.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/RemoteServiceCallImpl.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/RemoteServiceCallImpl.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -29,7 +29,6 @@
import java.util.UUID;
-import org.apache.http.HttpResponse;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.openrdf.query.BindingSet;
@@ -49,8 +48,6 @@
* adjusting the {@link RemoteServiceOptions} for the service URI.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id: RemoteServiceCallImpl.java 6060 2012-03-02 16:07:38Z
- * thompsonbry $
*/
public class RemoteServiceCallImpl implements RemoteServiceCall {
@@ -149,7 +146,9 @@
// Setup a standard strategy for following redirects.
httpClient.setRedirectStrategy(new DefaultRedirectStrategy());
- final RemoteRepository repo = new RemoteRepository(uriStr,//
+ final RemoteRepository repo = new RemoteRepository(//
+ uriStr,//
+ params.getServiceOptions().isBigdataLBS(),// useLBS
httpClient,//
params.getTripleStore().getExecutorService()
);
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceCallCreateParams.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceCallCreateParams.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceCallCreateParams.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -37,7 +37,6 @@
* {@link ServiceCall} instance.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public interface ServiceCallCreateParams {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceOptionsBase.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceOptionsBase.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceOptionsBase.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -29,19 +29,27 @@
/**
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
abstract public class ServiceOptionsBase implements IServiceOptions {
private boolean isRunFirst = false;
-
+ private boolean useLBS = false;
+
@Override
public boolean isRunFirst() {
return isRunFirst;
}
-
- public void setRunFirst(final boolean newValue) {
+
+ public void setRunFirst(final boolean newValue) {
this.isRunFirst = newValue;
}
+ @Override
+ public boolean isBigdataLBS() {
+ return useLBS;
+ }
+
+ public void setBigdataLBS(final boolean newValue) {
+ this.useLBS = newValue;
+ }
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailFactory.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailFactory.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailFactory.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -96,6 +96,10 @@
/**
* Connect to a remote bigdata instance.
+ *
+ * FIXME This does not support the HA load balancer pattern and does not
+ * parameterize the value of the ContextPath. Also, should this class be
+ * part of the "client" package?
*/
public static BigdataSailRemoteRepository connect(final String serviceEndpoint) {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/remote/BigdataSailRemoteRepository.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/remote/BigdataSailRemoteRepository.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/remote/BigdataSailRemoteRepository.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -1,5 +1,3 @@
-package com.bigdata.rdf.sail.remote;
-
/**
Copyright (C) SYSTAP, LLC 2006-2012. All rights reserved.
@@ -24,6 +22,8 @@
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
+package com.bigdata.rdf.sail.remote;
+
import java.io.File;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -66,9 +66,30 @@
/**
* Ctor that simply specifies an endpoint and lets this class manage the
* ClientConnectionManager for the HTTP client and the manage the
- * ExecutorService. More convenient.
+ * ExecutorService. More convenient, but does not account for whether or not
+ * to use the LBS.
+ *
+ * @param sparqlEndpointURL
+ * The SPARQL end point URL
*/
- public BigdataSailRemoteRepository(final String sparqlEndpointURL) {
+ public BigdataSailRemoteRepository(final String sparqlEndpointURL) {
+
+ this(sparqlEndpointURL, true/* useLBS */);
+
+ }
+
+ /**
+ * Ctor that simply specifies an endpoint and lets this class manage the
+ * ClientConnectionManager for the HTTP client and the manage the
+ * ExecutorService.
+ *
+ * @param sparqlEndpointURL
+ * The SPARQL end point URL
+ * @param useLBS
+ * <code>true</code> iff the LBS pattern should be used.
+ */
+ public BigdataSailRemoteRepository(final String sparqlEndpointURL,
+ final boolean useLBS) {
this.executor = Executors.newCachedThreadPool();
@@ -84,9 +105,9 @@
*/
httpClient.setRedirectStrategy(new DefaultRedirectStrategy());
- this.nss = new RemoteRepository(
- sparqlEndpointURL, httpClient, executor);
-
+ this.nss = new RemoteRepository(sparqlEndpointURL, useLBS, httpClient,
+ executor);
+
}
/**
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -386,8 +386,15 @@
* {@inheritDoc}
* <p>
* Extended to setup the as-configured {@link IHALoadBalancerPolicy}.
+ * <p>
+ * Note: If the deployment is does not support HA replication (e.g., either
+ * not HA or HA with replicationFactor:=1), then we still want to be able to
+ * forward to the local service.
*
* @throws ServletException
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/965" > Cannot run queries in
+ * LBS mode with HA1 setup </a>
*/
@Override
public void init() throws ServletException {
@@ -405,42 +412,35 @@
final IIndexManager indexManager = BigdataServlet
.getIndexManager(servletContext);
- if (!(indexManager instanceof HAJournal)){
- // This is not an error, but the LBS is only for HA.
- log.info("LBS Disabled - not HA");
- return;
- }
- if (indexManager instanceof AbstractJournal
+ if (indexManager instanceof HAJournal
&& ((AbstractJournal) indexManager).getQuorum() != null
&& ((AbstractJournal) indexManager).getQuorum()
- .replicationFactor() == 1) {
- // This is not an error, but the LBS is only for HA.
- log.info("LBS Disabled - not HA");
- return;
- }
+ .replicationFactor() > 1) {
- {
- // Get the as-configured policy.
- final IHALoadBalancerPolicy policy = newInstance(//
- servletConfig, //
- HALoadBalancerServlet.class,// owningClass
- IHALoadBalancerPolicy.class,//
- InitParams.POLICY, InitParams.DEFAULT_POLICY);
+ {
+ // Get the as-configured policy.
+ final IHALoadBalancerPolicy policy = newInstance(//
+ servletConfig, //
+ HALoadBalancerServlet.class,// owningClass
+ IHALoadBalancerPolicy.class,//
+ InitParams.POLICY, InitParams.DEFAULT_POLICY);
- // Set the as-configured policy.
- setLBSPolicy(policy);
+ // Set the as-configured policy.
+ setLBSPolicy(policy);
- }
- {
+ }
+ {
- final IHARequestURIRewriter rewriter = newInstance(//
- servletConfig,//
- HALoadBalancerServlet.class, // owningClass
- IHARequestURIRewriter.class,//
- InitParams.REWRITER, InitParams.DEFAULT_REWRITER);
+ final IHARequestURIRewriter rewriter = newInstance(//
+ servletConfig,//
+ HALoadBalancerServlet.class, // owningClass
+ IHARequestURIRewriter.class,//
+ InitParams.REWRITER, InitParams.DEFAULT_REWRITER);
- setRewriter(rewriter);
+ setRewriter(rewriter);
+ }
+
}
servletContext.setAttribute(BigdataServlet.ATTRIBUTE_LBS_PREFIX,
@@ -850,6 +850,10 @@
* the request to the servlet at the resulting requestURI. This forwarding
* effectively disables the LBS but still allows requests which target the
* LBS to succeed against the webapp on the same host.
+ * <p>
+ * Note: If the deployment is does not support HA replication (e.g., either
+ * not HA or HA with replicationFactor:=1), then we still want to be able to
+ * forward to the local service.
*
* @param request
* The request.
@@ -858,6 +862,9 @@
*
* @throws IOException
* @throws ServletException
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/965" > Cannot run queries in
+ * LBS mode with HA1 setup </a>
*/
public void forwardToLocalService(//
final boolean isLeaderRequest,//
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepository.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepository.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepository.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -307,6 +307,20 @@
}
+ /**
+ *
+ * @param sparqlEndpointURL
+ * @param httpClient
+ * @param executor
+ *
+ * @deprecated This version does not force the caller to decide whether or
+ * not the LBS pattern will be used. In general, it should be
+ * used if the end point is bigdata. This class is generally,
+ * but not always, used with a bigdata end point. The main
+ * exception is SPARQL Basic Federated Query. For that use case
+ * we can not assume that the end point is bigdata and thus we
+ * can not use the LBS prefix.
+ */
public RemoteRepository(final String sparqlEndpointURL,
final HttpClient httpClient, final Executor executor) {
@@ -865,7 +879,7 @@
}
if (add.uri != null) {
- // set the resource to load.
+ // set the resource to load : FIXME REST API allows multiple URIs, but RemoteRepository does not.
opts.addRequestParam("uri", add.uri);
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepositoryManager.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepositoryManager.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepositoryManager.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -89,7 +89,7 @@
public RemoteRepositoryManager(final String serviceURL,
final HttpClient httpClient, final Executor executor) {
- this(serviceURL, false/* useLBS */, httpClient, executor);
+ this(serviceURL, true/* useLBS */, httpClient, executor);
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/health/TestNSSHealthCheck.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/health/TestNSSHealthCheck.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/health/TestNSSHealthCheck.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -375,9 +375,9 @@
*
* @return The connection.
*
- * @see <a href="https://sourceforge.net/apps/trac/bigdata/ticket/619">
- * RemoteRepository class should use application/x-www-form-urlencoded
- * for large POST requests </a>
+ * @see <a href="http://trac.bigdata.com/ticket/619"> RemoteRepository class
+ * should use application/x-www-form-urlencoded for large POST requests
+ * </a>
*/
private HttpResponse doConnect(final ConnectOptions opts) throws Exception {
@@ -452,57 +452,17 @@
}
-// // conn = doConnect(urlString.toString(), opts.method);
-// final URL url = new URL(urlString.toString());
-// conn = (HttpURLConnection) url.openConnection();
-// conn.setRequestMethod(opts.method);
-// conn.setDoOutput(true);
-// conn.setDoInput(true);
-// conn.setUseCaches(false);
-// conn.setReadTimeout(opts.timeout);
-// conn.setRequestProperty("Accept", opts.acceptHeader);
-// if (log.isDebugEnabled())
-// log.debug("Accept: " + opts.acceptHeader);
-
if (opts.entity != null) {
-// if (opts.data == null)
-// throw new AssertionError();
-
-// final String contentLength = Integer.toString(opts.data.length);
-
-// conn.setRequestProperty("Content-Type", opts.contentType);
-// conn.setRequestProperty("Content-Length", contentLength);
-
-// if (log.isDebugEnabled()) {
-// log.debug("Content-Type: " + opts.contentType);
-// log.debug("Content-Length: " + contentLength);
-// }
-
-// final ByteArrayEntity entity = new ByteArrayEntity(opts.data);
-// entity.setContentType(opts.contentType);
-
- ((HttpEntityEnclosingRequestBase) request).setEntity(opts.entity);
+ ((HttpEntityEnclosingRequestBase) request)
+ .setEntity(opts.entity);
-// final OutputStream os = conn.getOutputStream();
-// try {
-// os.write(opts.data);
-// os.flush();
-// } finally {
-// os.close();
-// }
-
}
final HttpResponse response = m_httpClient.execute(request);
return response;
-// // connect.
-// conn.connect();
-//
-// return conn;
-
} catch (Throwable t) {
/*
* If something goes wrong, then close the http connection.
@@ -513,10 +473,6 @@
if (request != null)
request.abort();
-// // clean up the connection resources
-// if (conn != null)
-// conn.disconnect();
-
} catch (Throwable t2) {
// ignored.
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
