Menu
▾
▴
bigdata-commit — Commit traffic.
This list is closed, nobody may subscribe to it.
| 2010 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
(139) |
Aug
(94) |
Sep
(232) |
Oct
(143) |
Nov
(138) |
Dec
(55) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2011 |
Jan
(127) |
Feb
(90) |
Mar
(101) |
Apr
(74) |
May
(148) |
Jun
(241) |
Jul
(169) |
Aug
(121) |
Sep
(157) |
Oct
(199) |
Nov
(281) |
Dec
(75) |
| 2012 |
Jan
(107) |
Feb
(122) |
Mar
(184) |
Apr
(73) |
May
(14) |
Jun
(49) |
Jul
(26) |
Aug
(103) |
Sep
(133) |
Oct
(61) |
Nov
(51) |
Dec
(55) |
| 2013 |
Jan
(59) |
Feb
(72) |
Mar
(99) |
Apr
(62) |
May
(92) |
Jun
(19) |
Jul
(31) |
Aug
(138) |
Sep
(47) |
Oct
(83) |
Nov
(95) |
Dec
(111) |
| 2014 |
Jan
(125) |
Feb
(60) |
Mar
(119) |
Apr
(136) |
May
(270) |
Jun
(83) |
Jul
(88) |
Aug
(30) |
Sep
(47) |
Oct
(27) |
Nov
(23) |
Dec
|
| 2015 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
(3) |
Oct
|
Nov
|
Dec
|
| 2016 |
Jan
|
Feb
|
Mar
(4) |
Apr
(1) |
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
|
From: <tho...@us...> - 2014-06-30 13:03:23
|
Revision: 8509
http://sourceforge.net/p/bigdata/code/8509
Author: thompsonbry
Date: 2014-06-30 13:03:14 +0000 (Mon, 30 Jun 2014)
Log Message:
-----------
Modified the Banner class to provide more transparent access to the BuildInfo metadata.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/Banner.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/Banner.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/Banner.java 2014-06-30 11:32:23 UTC (rev 8508)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/Banner.java 2014-06-30 13:03:14 UTC (rev 8509)
@@ -249,23 +249,52 @@
}
/**
- * Use reflection to discover and report on the bigdata build information. A
- * <code>com.bigdata.BuildInfo</code> is built when the JAR is created.
- * However, it may not be present when running under an IDE from the source
- * code and, therefore, there MUST NOT be any compile time references to the
- * <code>com.bigdata.BuildInfo</code> class. This method uses reflection to
- * avoid a compile time dependency.
+ * An interface which declares the keys for the map returned by
+ * {@link Banner#getBuildInfo()} .
+ *
+ * @author <a href="mailto:tho...@us...">Bryan
+ * Thompson</a>
+ */
+ public interface BuildInfoMeta {
+ /** The bigdata release version. */
+ String buildVersion = "buildVersion";
+ /** The source code revision. */
+ String svnRevision = "svnRevision";
+ /** The source code repository URL for the branch. */
+ String svnURL = "svnURL";
+ /** The timestamp of the build. */
+ String buildTimestamp = "buildTimestamp";
+ /** The username that performed the build. */
+ String buildUser = "buildUser";
+ /** The hostname on which the build was performed. */
+ String buildHost = "buildHost";
+ /** The OS architecture on which the build was performed. */
+ String osArch = "osArch";
+ /** The OS name on which the build was performed. */
+ String osName = "osName";
+ /** The OS version on which the build was performed. */
+ String osVersion = "osVersion";
+ }
+
+ /**
+ * Method used to discover and report on the bigdata build information. A
+ * <code>com.bigdata.BuildInfo</code> class is built when the JAR is
+ * created. However, it may not be present when running under an IDE from
+ * the source code and, therefore, there MUST NOT be any compile time
+ * references to the <code>com.bigdata.BuildInfo</code> class. This method
+ * uses reflection to avoid a compile time dependency.
* <p>
- * Note: This method works fine. However, the problem with exposing the
- * information is that people running from an IDE can observe <em>stale</em>
- * data from old <code>com.bigdata.BuildInfo</code> class files left from a
- * previous build of a JAR. This makes the information good for deployed
- * versions of the JAR but potentially misleading when people are running
- * under an IDE.
+ * Note: This method works fine. However, people running from an IDE will
+ * observe <em>stale</em> data from old <code>com.bigdata.BuildInfo</code>
+ * class files left from a previous build of a JAR. This makes the
+ * information good for deployed versions of the JAR but potentially
+ * misleading when people are running under an IDE.
*
* @return Build info metadata iff available.
+ *
+ * @see BuildInfoMeta
*/
- private synchronized static Map<String,String> getBuildInfo() {
+ public synchronized static Map<String,String> getBuildInfo() {
if (buildInfoRef.get() == null) {
@@ -341,7 +370,7 @@
final StringBuilder s = new StringBuilder();
- s.append("\nbuildVersion=" + getBuildInfo().get("buildVersion"));
+ s.append("\nbuildVersion=" + getBuildInfo().get(BuildInfoMeta.buildVersion));
// s.append("\nsvnRevision =" + getBuildInfo().get("svnRevision"));
@@ -350,16 +379,25 @@
}
/**
- * Attempts to return build version.
+ * Attempts to return the build version (aka the release version) from the
+ * <code>com.bigdata.BuildInfo</code> class. This class is generated by
+ * <code>build.xml</code> and is NOT available from the IDE. It is correct
+ * discovered using reflection.
*
- * @return Build version if available
+ * @return Build version if available and <code>unknown</code> otherwise.
+ *
+ * @see #getBuildInfo()
*/
public final static String getVersion() {
- if (getBuildInfo().isEmpty()) {
- return "unknown";
- }
-
- return getBuildInfo().get("buildVersion");
+
+ if (getBuildInfo().isEmpty()) {
+
+ return "unknown";
+
+ }
+
+ return getBuildInfo().get(BuildInfoMeta.buildVersion);
+
}
/**
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2014-06-30 11:32:35
|
Revision: 8508
http://sourceforge.net/p/bigdata/code/8508
Author: martyncutcher
Date: 2014-06-30 11:32:23 +0000 (Mon, 30 Jun 2014)
Log Message:
-----------
For ticket #936, modified metabits demispace use to default to "false" and support an explicit, programmatic, request to toggle between fixed allocation and demispace.
Provide a utility MetabitsUtil class with "main" to toggle from a command line invocation.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWStore.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/MetabitsUtil.java
Added: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/MetabitsUtil.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/MetabitsUtil.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/MetabitsUtil.java 2014-06-30 11:32:23 UTC (rev 8508)
@@ -0,0 +1,105 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2014. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package com.bigdata.rwstore;
+
+import java.io.File;
+import java.util.Properties;
+
+import com.bigdata.journal.BufferMode;
+import com.bigdata.journal.RWStrategy;
+import com.bigdata.journal.Journal;
+import com.bigdata.journal.Journal.Options;
+import com.bigdata.rawstore.IRawStore;
+
+/**
+ * A utility class to explicitly change the metabits storage to allow for
+ * compatibility with previous versions.
+ * <p>
+ * There is an option to use a demispace rather than standard allocations to
+ * support stores with large numbers of allocations. If such a store needs to be
+ * opened by an earlier code-base, then the store must be amended to store the
+ * metabits in a standard allocation.
+ *
+ * @author Martyn Cutcher
+ *
+ */
+public class MetabitsUtil {
+
+ static String getArg(final String[] args, final String arg, final String def) {
+ for (int p = 0; p < args.length; p += 2) {
+ if (arg.equals(args[p]))
+ return args[p + 1];
+ }
+
+ return def;
+ }
+
+ static Journal getStore(String storeFile) {
+
+ final Properties properties = new Properties();
+
+ properties.setProperty(Options.FILE, storeFile);
+
+ properties.setProperty(Options.BUFFER_MODE,
+ BufferMode.DiskRW.toString());
+
+ return new Journal(properties);// .getBufferStrategy();
+
+ }
+
+ /**
+ * Example usage:
+ * <p>
+ * MatabitsUtil -store "/path/store.jnl" -usedemispace true
+ */
+ static public void main(final String[] args) {
+ final String store = getArg(args, "-store", null);
+ if (store == null) {
+ System.err.println("file must be specificed with -store");
+ return;
+ }
+ final File file = new File(store);
+ if (!file.exists()) {
+ System.err.println("Specified file '" + store + "' not found");
+ return;
+ }
+
+ final boolean usedemi = "true".equals(getArg(args, "-usedemispace",
+ "true"));
+
+ final Journal jnl = getStore(store);
+
+ try {
+ final RWStore rws = ((RWStrategy) jnl.getBufferStrategy())
+ .getStore();
+
+ if (rws.ensureMetabitsDemispace(usedemi)) { // changed
+ jnl.commit();
+ }
+ } finally {
+ jnl.close();
+ }
+ }
+
+}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2014-06-28 01:45:04 UTC (rev 8507)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2014-06-30 11:32:23 UTC (rev 8508)
@@ -342,7 +342,7 @@
*/
String META_BITS_DEMI_SPACE = RWStore.class.getName() + ".metabitsDemispace";
- String DEFAULT_META_BITS_DEMI_SPACE = "true";
+ String DEFAULT_META_BITS_DEMI_SPACE = "false";
/**
* Defines the number of bits that must be free in a FixedAllocator for
@@ -1478,11 +1478,17 @@
// Can handle minor store version incompatibility
final int storeVersion = strBuf.readInt();
- if ((storeVersion & 0xFF00) != (cVersion & 0xFF00)) {
+
+ switch ((storeVersion & 0xFF00)) {
+ case (cVersion & 0xFF00):
+ case (cVersionDemispace & 0xFF00):
+ break;
+ default:
throw new IllegalStateException(
"Incompatible RWStore header version: storeVersion="
+ storeVersion + ", cVersion=" + cVersion);
}
+
m_lastDeferredReleaseTime = strBuf.readLong();
if (strBuf.readInt() != cDefaultMetaBitsSize) {
throw new IllegalStateException("Store opened with unsupported metabits size");
@@ -3031,7 +3037,7 @@
final FixedOutputStream str = new FixedOutputStream(buf);
try {
- str.writeInt(cVersion);
+ str.writeInt(m_metaBitsAddr > 0 ? cVersionDemispace : cVersion);
str.writeLong(m_lastDeferredReleaseTime);
str.writeInt(cDefaultMetaBitsSize);
str.writeInt(m_allocSizes.length);
@@ -3068,7 +3074,7 @@
* writeMetaBits().
*/
//final long addr = physicalAddress(m_metaBitsAddr);
- final long addr = ((long) m_metaBitsAddr) << ALLOCATION_SCALEUP;
+ final long addr = m_metaBitsAddr < 0 ? physicalAddress(m_metaBitsAddr) : ((long) m_metaBitsAddr) << ALLOCATION_SCALEUP;
if (addr == 0) {
throw new IllegalStateException("Invalid metabits address: " + m_metaBitsAddr);
}
@@ -3081,7 +3087,7 @@
// Similar to writeMetaBits, we are no longer writing to a FixedAllocator managed region,
// so no latched address is provided
- m_writeCacheService.write(addr, ByteBuffer.wrap(buf), 0/*chk*/, false/*useChecksum*/, 0 /*latchedAddr*/);
+ m_writeCacheService.write(addr, ByteBuffer.wrap(buf), 0/*chk*/, false/*useChecksum*/, m_metaBitsAddr < 0 ? m_metaBitsAddr : 0 /*latchedAddr*/);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
@@ -3134,42 +3140,69 @@
* that we do not need to reallocate the metabits region when we are
* writing out the updated versions of the FixedAllocators).
*/
-// final long oldMetaBits = m_metaBitsAddr;
-// final int oldMetaBitsSize = (m_metaBits.length + m_allocSizes.length + 1) * 4;
-// m_metaBitsAddr = alloc(getRequiredMetaBitsStorage(), null);
+ if (m_metaBitsAddr > 0) {
+ // already using demi-space, remove from WCS
+ m_writeCacheService.removeWriteToAddr(convertAddr(-m_metaBitsAddr), 0);
+ } else {
+ final int reqmbc = getRequiredMetaBitsStorage();
+ int nmbaddr = 0;
+ // if > max alloc or explicitly use the demi-space, then drop through for demi-space
+ if ((!m_useMetabitsDemispace) && reqmbc < m_maxFixedAlloc) {
+ nmbaddr = alloc(reqmbc, null);
+ }
- /*
- * If m_metaBitsAddr < 0 then was allocated from FixedAllocators (for existing-store compatibility)
- */
- if (m_metaBitsAddr < 0) {
- if (physicalAddress(m_metaBitsAddr) == 0) {
- throw new IllegalStateException("Returned MetaBits Address not valid!");
- }
-
- final int oldMetaBitsSize = (m_metaBits.length + m_allocSizes.length + 1) * 4;
- // Call immediateFree - no need to defer freeof metaBits, this
- // has to stop somewhere!
- // No more allocations must be made
- immediateFree((int) m_metaBitsAddr, oldMetaBitsSize);
-
- m_metaBitsAddr = 0;
- }
-
+ // If existing allocation, then free it
+ if (m_metaBitsAddr < 0) {
+
+ final int oldMetaBitsSize = (m_metaBits.length
+ + m_allocSizes.length + 1) * 4;
+
+ // Call immediateFree - no need to defer freeof metaBits, this
+ // has to stop somewhere!
+ // No more allocations must be made
+ immediateFree((int) m_metaBitsAddr, oldMetaBitsSize);
+
+ }
+
+ m_metaBitsAddr = nmbaddr;
+ }
+
if (m_metaBitsAddr == 0) {
// Allocate special region to be able to store maximum metabits (128k of 2 64K demi-space
// Must be aligned on 128K boundary and allocations are made in units of 64K.
+ //
+ // May need to extend the file for teh demi-space!
while (m_nextAllocation % 2 != 0) {
m_nextAllocation--;
}
m_metaBitsAddr = -m_nextAllocation; // must be positive to differentiate from FixedAllocator address
m_nextAllocation -= 2; // allocate 2 * 64K
- } else { // remove previous write from WCS
- m_writeCacheService.removeWriteToAddr(convertAddr(-m_metaBitsAddr), 0);
+
+ // Check for file extension
+ while (m_nextAllocation <= m_fileSize) {
+ extendFile();
+ }
+
+ if (log.isInfoEnabled())
+ log.info("Using Demi-space metabits");
}
- // Now "toggle" m_metaBitsAddr - 64K boundary
- m_metaBitsAddr ^= 0x01; // toggle zero or 64K offset
+ if (m_metaBitsAddr > 0) { // Demi-Space
+ // Now "toggle" m_metaBitsAddr - 64K boundary
+ m_metaBitsAddr ^= 0x01; // toggle zero or 64K offset
+ }
+ if (log.isDebugEnabled()) {
+ final long mbaddr;
+ if (m_metaBitsAddr < 0) {
+ mbaddr = physicalAddress((int) m_metaBitsAddr);
+ } else {
+ mbaddr = convertAddr(-m_metaBitsAddr); // maximum 48 bit address range
+ }
+
+ log.debug("Writing metabits at " + mbaddr);
+ }
+
// There must be no buffered deferred frees
// assert m_deferredFreeOut.getBytesWritten() == 0;
@@ -3451,8 +3484,10 @@
* Versions
* 0x0300 - extended header to include reserved ints
* 0x0400 - removed explicit BlobAllocators
+ * 0x0500 - using metaBits demi-space
*/
final private int cVersion = 0x0400;
+ final private int cVersionDemispace = 0x0500;
/**
* cReservedMetaBits is the reserved space in the metaBits header
@@ -4233,31 +4268,41 @@
*
* @return long representation of metaBitsAddr PLUS the size
*/
- public long getMetaBitsAddr() {
- assert m_metaBitsAddr > 0;
-
- // long ret = physicalAddress((int) m_metaBitsAddr);
- long ret = convertAddr(-m_metaBitsAddr); // maximum 48 bit address range
- ret <<= 16;
-
- // include space for version, allocSizes and deferred free info AND cDefaultMetaBitsSize
- final int metaBitsSize = cMetaHdrFields + m_metaBits.length + m_allocSizes.length;
- ret += metaBitsSize;
-
- if (log.isTraceEnabled())
- log.trace("Returning metabitsAddr: " + ret + ", for "
- + m_metaBitsAddr + " - " + m_metaBits.length + ", "
- + metaBitsSize);
+ public long getMetaBitsAddr() {
+ long ret = 0;
- return ret;
- }
+ if (m_metaBitsAddr < 0) {
+ ret = physicalAddress((int) m_metaBitsAddr);
+ } else {
+ // long ret = physicalAddress((int) m_metaBitsAddr);
+ ret = convertAddr(-m_metaBitsAddr); // maximum 48 bit address range
+ }
+ ret <<= 16;
+ // include space for version, allocSizes and deferred free info AND
+ // cDefaultMetaBitsSize
+ final int metaBitsSize = cMetaHdrFields + m_metaBits.length
+ + m_allocSizes.length;
+ ret += metaBitsSize;
+
+ if (log.isTraceEnabled())
+ log.trace("Returning metabitsAddr: " + ret + ", for "
+ + m_metaBitsAddr + " - " + m_metaBits.length + ", "
+ + metaBitsSize);
+
+ return ret;
+ }
+
/**
*
- * @return the address of the metaBits demi-space
+ * @return the address of the metaBits
*/
- public long getMetaBitsDemiSpace() {
- return convertAddr(-m_metaBitsAddr);
+ public long getMetaBitsStoreAddress() {
+ if (m_metaBitsAddr < 0) {
+ return physicalAddress((int) m_metaBitsAddr);
+ } else {
+ return convertAddr(-m_metaBitsAddr); // maximum 48 bit address range
+ }
}
/**
@@ -7188,6 +7233,31 @@
return ret;
}
+
+ /**
+ * Forces a reset of the metabits allocation on the next commit.
+ * <p>
+ * Note that a side-effect of this is that there will be a memory leak
+ * of either a FixedAllocation slot or an existing demi-space.
+ * <p>
+ * @param useDemispace
+ * @return whether the storage has been modified.
+ */
+ public boolean ensureMetabitsDemispace(final boolean useDemispace) {
+ final boolean isDemispace = m_metaBitsAddr > 0;
+
+ if (isDemispace != useDemispace || m_useMetabitsDemispace != useDemispace) {
+ m_useMetabitsDemispace = useDemispace;
+
+ m_metaBitsAddr = 0;
+
+ m_recentAlloc = true; // force commit
+
+ return true;
+ } else {
+ return false;
+ }
+ }
// public void prepareForRebuild(final HARebuildRequest req) {
// assert m_rebuildRequest == null;
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java 2014-06-28 01:45:04 UTC (rev 8507)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java 2014-06-30 11:32:23 UTC (rev 8508)
@@ -1540,23 +1540,116 @@
public void test_metaAlloc() {
Journal store = (Journal) getStore();
- try {
+ try {
- final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
+ final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
- final RWStore rw = bs.getStore();
- long realAddr = 0;
- for (int i = 0; i < 100000; i++) {
- int allocAddr = rw.metaAlloc();
+ final RWStore rw = bs.getStore();
+ long realAddr = 0;
+ for (int r = 0; r < 100; r++) {
+ for (int i = 0; i < 1000; i++) {
+ int allocAddr = rw.metaAlloc();
+
+ realAddr = rw.metaBit2Addr(allocAddr);
+ }
+ rw.commit();
+ }
- realAddr = rw.metaBit2Addr(allocAddr);
- }
- if(log.isInfoEnabled())log.info("metaAlloc lastAddr: " + realAddr);
+ if (log.isInfoEnabled())
+ log.info("metaAlloc lastAddr: " + realAddr);
} finally {
store.destroy();
}
}
+
+ /**
+ * Tests the MetabitsUtil to switch the demispace.
+ *
+ * If the address is a demispace then addr % 64K == 0.
+ *
+ * If the address is NOT a demispace then it should be less than first
+ * demispace
+ */
+ public void test_metabitsDemispace() {
+ Journal store = (Journal) getStore();
+ try {
+ RWStrategy bs = (RWStrategy) store.getBufferStrategy();
+ RWStore rw = bs.getStore();
+ final String fname = rw.getStoreFile().getAbsolutePath();
+
+ store.commit();
+
+ final long fa1 = rw.getMetaBitsStoreAddress();
+
+ rw.ensureMetabitsDemispace(true);
+ store.commit();
+
+ final long ds1 = rw.getMetaBitsStoreAddress();
+
+ assertTrue((ds1 & 0xFFFF) == 0); // MOD 64K
+ assertTrue(ds1 > fa1);
+
+ rw.ensureMetabitsDemispace(false);
+ store.commit();
+
+ final long fa2 = rw.getMetaBitsStoreAddress();
+
+ assertTrue(ds1 > fa2);
+
+ rw.ensureMetabitsDemispace(true);
+ store.commit();
+
+ final long ds2 = rw.getMetaBitsStoreAddress();
+
+ assertTrue((ds2 & 0xFFFF) == 0);
+ assertTrue(ds2 > ds1);
+
+ // Now use MetaBitsUtil
+
+ store.close();
+
+ MetabitsUtil.main(new String[] { "-store", fname, "-usedemispace", "false"});
+
+ store = getExplicitStore(fname);
+
+ bs = (RWStrategy) store.getBufferStrategy();
+ rw = bs.getStore();
+ final long fa3 = rw.getMetaBitsStoreAddress();
+
+ assertTrue(fa3 < ds1);
+
+ store.close();
+
+ MetabitsUtil.main(new String[] { "-store", fname, "-usedemispace", "true"});
+
+ store = getExplicitStore(fname);
+
+ bs = (RWStrategy) store.getBufferStrategy();
+ rw = bs.getStore();
+
+ final long ds3 = rw.getMetaBitsStoreAddress();
+ assertTrue((ds3 & 0xFFFF) == 0);
+ assertTrue(ds3 > ds2);
+
+ } finally {
+ store.destroy();
+ }
+ }
+
+ Journal getExplicitStore(String storeFile) {
+
+ final Properties properties = new Properties();
+
+ properties.setProperty(Options.FILE, storeFile);
+
+ properties.setProperty(Options.BUFFER_MODE,
+ BufferMode.DiskRW.toString());
+
+ return new Journal(properties);// .getBufferStrategy();
+
+ }
+
static class DummyAllocationContext implements IAllocationContext {
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-06-28 01:45:10
|
Revision: 8507
http://sourceforge.net/p/bigdata/code/8507
Author: tobycraig
Date: 2014-06-28 01:45:04 +0000 (Sat, 28 Jun 2014)
Log Message:
-----------
Added per service status to health tab
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/Banner.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HAStatusServletUtil.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/Banner.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/Banner.java 2014-06-27 21:26:52 UTC (rev 8506)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/Banner.java 2014-06-28 01:45:04 UTC (rev 8507)
@@ -350,6 +350,19 @@
}
/**
+ * Attempts to return build version.
+ *
+ * @return Build version if available
+ */
+ public final static String getVersion() {
+ if (getBuildInfo().isEmpty()) {
+ return "unknown";
+ }
+
+ return getBuildInfo().get("buildVersion");
+ }
+
+ /**
* Outputs the banner and exits.
*
* @param args
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HAStatusServletUtil.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HAStatusServletUtil.java 2014-06-27 21:26:52 UTC (rev 8506)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HAStatusServletUtil.java 2014-06-28 01:45:04 UTC (rev 8507)
@@ -43,6 +43,7 @@
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooKeeper;
+import com.bigdata.Banner;
import com.bigdata.BigdataStatics;
import com.bigdata.ha.HAGlue;
import com.bigdata.ha.HAStatusEnum;
@@ -916,28 +917,39 @@
json.writeStartObject();
- json.writeFieldName("version");
- json.writeString("1.0"); // FIXME
- json.writeFieldName("timestamp");
- json.writeNumber(new Date().getTime()); // FIXME
+ json.writeStringField("version", Banner.getVersion());
+ json.writeNumberField("timestamp", new Date().getTime());
if(quorum.isQuorumFullyMet(quorum.token())) {
- json.writeFieldName("status");
- json.writeString("Good");
- json.writeFieldName("details");
- json.writeString("All servers joined");
+ json.writeStringField("status", "Good");
+ json.writeStringField("details", "All servers joined");
} else {
// at least one server is not available, so status is either Warning or Bad
- json.writeFieldName("status");
if(quorum.isQuorumMet()) {
- json.writeString("Warning");
+ json.writeStringField("status", "Warning");
} else {
- json.writeString("Bad");
+ json.writeStringField("status", "Bad");
}
- json.writeFieldName("details");
- json.writeString("Only " + quorum.getJoined().length + " of target " +
+ json.writeStringField("details", "Only " + quorum.getJoined().length + " of target " +
quorum.replicationFactor() + " servers joined");
}
+ json.writeFieldName("services");
+ json.writeStartArray();
+
+ final UUID[] joined = quorum.getJoined();
+ final UUID[] pipeline = quorum.getPipeline();
+
+ for (UUID serviceId : pipeline) {
+ final boolean isLeader = serviceId.equals(quorum.getLeaderId());
+ final boolean isFollower = indexOf(serviceId, joined) > 0;
+
+ json.writeStartObject();
+ json.writeStringField("id", serviceId.toString());
+ json.writeStringField("status", isLeader ? "leader" : (isFollower ? "follower" : "unready"));
+ json.writeEndObject();
+ }
+
+ json.writeEndArray();
json.writeEndObject();
json.close();
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-06-27 21:26:52 UTC (rev 8506)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-06-28 01:45:04 UTC (rev 8507)
@@ -202,11 +202,17 @@
<div class="tab" id="health-tab">
+ <div class="box" id="health-overview">
+ <h1>Overview</h1>
+ <p class="health-status">Status: <span></span></p>
+ <p class="health-details">Details: <span></span></p>
+ <p class="health-version">Version: <span></span></p>
+ <p class="health-timestamp">Timestamp: <span></span></p>
+ </div>
+
+ <div id="health-services"></div>
+
<div class="box">
- <p id="health-status">Status: <span></span></p>
- <p id="health-details">Details: <span></span></p>
- <p id="health-version">Version: <span></span></p>
- <p id="health-timestamp">Timestamp: <span></span></p>
<p><a href="#" id="health-refresh">Refresh</a></p>
</div>
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-27 21:26:52 UTC (rev 8506)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-28 01:45:04 UTC (rev 8507)
@@ -1406,15 +1406,31 @@
function getHealth(e) {
e.preventDefault();
$.get('/status?health', function(data) {
- for(var key in data) {
- if(key == 'timestamp') {
- var date = new Date(data[key]);
- data[key] = date.toString();
+ $('#health-overview .health-status span').html(data.status);
+ $('#health-overview').removeClass('health-good health-warning health-bad').addClass('health-' + data.status.toLowerCase());
+ $('#health-overview .health-details span').html(data.details);
+ $('#health-overview .health-version span').html(data.version);
+ $('#health-overview .health-timestamp span').html(new Date(data.timestamp).toString());
+
+ $('#health-services div').remove();
+ for(var i=0; i<data.services.length; i++) {
+ var div = $('<div>');
+ div.append('<p>ID: ' + data.services[i].id + '</p>');
+ div.append('<p>Status: ' + data.services[i].status + '</p>');
+ var health;
+ switch(data.services[i].status) {
+ case 'leader':
+ case 'follower':
+ health = 'good';
+ break;
+ case 'unready':
+ health = 'warning';
+ break;
+ default:
+ health = 'bad';
}
- if(key == 'status') {
- $('#health-overview').removeClass('health-good health-warning health-bad').addClass('health-' + data[key].toLowerCase());
- }
- $('#health-' + key + ' span').html(data[key]);
+ div.addClass('box health-' + health);
+ div.appendTo($('#health-services'));
}
})
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-06-27 21:26:57
|
Revision: 8506
http://sourceforge.net/p/bigdata/code/8506
Author: tobycraig
Date: 2014-06-27 21:26:52 +0000 (Fri, 27 Jun 2014)
Log Message:
-----------
Merged query history into 1.3 branch
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css 2014-06-26 22:47:58 UTC (rev 8505)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css 2014-06-27 21:26:52 UTC (rev 8506)
@@ -158,7 +158,6 @@
.modal {
display: none;
- z-index: 1;
position: fixed;
top: 0;
left: 0;
@@ -167,6 +166,7 @@
margin-left: 25%;
background-color: white;
padding: 20px;
+ z-index: 4;
}
#overlay {
@@ -178,6 +178,7 @@
height: 100%;
background-color: grey;
opacity: 0.5;
+ z-index: 3;
}
.modal-open #overlay {
@@ -248,7 +249,7 @@
border: none;
}
-.advanced-features, #query-response, #query-pagination, #query-explanation, #query-export-container, #update-response, #update-clear-container, #explore-results, #namespace-properties {
+.advanced-features, #query-response, #query-pagination, #query-explanation, #query-history, #query-export-container, #update-response, #update-clear-container, #explore-results, #namespace-properties {
display: none;
}
@@ -353,3 +354,8 @@
.health-bad {
background-color: red;
}
+
+#links {
+ text-align: center;
+ margin-top: 20px;
+}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-06-26 22:47:58 UTC (rev 8505)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-06-27 21:26:52 UTC (rev 8506)
@@ -135,6 +135,20 @@
<div id="query-explanation" class="box">
</div>
+ <div id="query-history" class="box">
+ <table>
+ <thead>
+ <tr>
+ <th>Time</th>
+ <th>Namespace</th>
+ <th>Query</th>
+ <th>Results</th>
+ </tr>
+ </thead>
+ <tbody></tbody>
+ </table>
+ </div>
+
<div id="query-export-container" class="box">
<button id="query-export">Export</button>
<button id="query-response-clear">Clear</button>
@@ -231,7 +245,7 @@
</div>
- <div class="clear"> </div>
+ <div id="links"><a href="http://www.bigdata.com" target="_blank">Bigdata</a> - <a href="http://wiki.bigdata.com/" target="_blank">Wiki</a></div>
</div>
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-26 22:47:58 UTC (rev 8505)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-27 21:26:52 UTC (rev 8506)
@@ -261,7 +261,7 @@
data: data,
contentType: 'application/xml',
success: function() { $('#new-namespace-name').val(''); getNamespaces(); },
- error: function(jqXHR, textStatus, errorThrown) { debugger;alert(jqXHR.responseText); }
+ error: function(jqXHR, textStatus, errorThrown) { alert(jqXHR.responseText); }
};
$.ajax(RW_URL_PREFIX + 'namespace', settings);
}
@@ -333,22 +333,28 @@
/* Update */
-function handleDragOver(e) {
+function handleDragOver(cm, e) {
e.stopPropagation();
e.preventDefault();
- e.originalEvent.dataTransfer.dropEffect = 'copy';
+ e.dataTransfer.dropEffect = 'copy';
}
-function handleFile(e) {
+function handleDrop(cm, e) {
e.stopPropagation();
e.preventDefault();
+ var files = e.dataTransfer.files;
+ handleFile(files);
+}
- if(e.type == 'drop') {
- var files = e.originalEvent.dataTransfer.files;
- } else {
- var files = e.originalEvent.target.files;
- }
-
+function handleFileInput(e) {
+ e.stopPropagation();
+ e.preventDefault();
+ var files = e.originalEvent.target.files;
+ handleFile(files);
+ $('#update-file').val('');
+}
+
+function handleFile(files) {
// only one file supported
if(files.length > 1) {
alert('Ignoring all but first file');
@@ -359,31 +365,29 @@
// if file is too large, tell user to supply local path
if(f.size > 1048576 * 100) {
alert('File too large, enter local path to file');
- $('#update-box').val('/path/to/' + f.name);
+ EDITORS.update.setValue('/path/to/' + f.name);
setType('path');
- $('#update-box').prop('disabled', false)
+ EDITORS.update.setOption('readOnly', false)
$('#large-file-message, #clear-file').hide();
} else {
var fr = new FileReader();
- fr.onload = function(e2) {
+ fr.onload = function(e) {
if(f.size > 10240) {
// do not use textarea
- $('#update-box').prop('disabled', true)
+ EDITORS.update.setOption('readOnly', true)
$('#filename').html(f.name);
$('#large-file-message, #clear-file').show()
- $('#update-box').val('');
- FILE_CONTENTS = e2.target.result;
+ EDITORS.update.setValue('');
+ FILE_CONTENTS = e.target.result;
} else {
// display file contents in the textarea
clearFile();
- $('#update-box').val(e2.target.result);
+ EDITORS.update.setValue(e.target.result);
}
- guessType(f.name.split('.').pop().toLowerCase(), e2.target.result);
+ guessType(f.name.split('.').pop().toLowerCase(), e.target.result);
};
fr.readAsText(f);
}
-
- $('#update-file').val('');
}
function clearFile(e) {
@@ -501,11 +505,11 @@
var sparql_update_commands = ['INSERT', 'DELETE', 'LOAD', 'CLEAR'];
-$('#update-file').change(handleFile);
-$('#update-box').on('dragover', handleDragOver)
- .on('drop', handleFile)
- .on('paste', handlePaste)
- .on('input propertychange', function() { $('#update-errors').hide(); });
+$('#update-file').change(handleFileInput);
+// $('#update-box').on('dragover', handleDragOver)
+// .on('drop', handleFile)
+// .on('paste', handlePaste)
+// .on('input propertychange', function() { $('#update-errors').hide(); });
$('#clear-file').click(clearFile);
$('#update-update').click(submitUpdate);
@@ -517,6 +521,9 @@
ERROR_CHARACTER_MARKERS.update.clear();
}
});
+EDITORS.update.on('dragover', handleDragOver);
+EDITORS.update.on('drop', handleDrop);
+EDITORS.update.on('paste', handlePaste);
EDITORS.update.addKeyMap({'Ctrl-Enter': submitUpdate});
function submitUpdate(e) {
@@ -645,6 +652,14 @@
});
EDITORS.query.addKeyMap({'Ctrl-Enter': submitQuery});
+$('#query-history').on('click', '.query', loadHistory);
+
+function loadHistory() {
+ EDITORS.query.setValue(this.innerText);
+ useNamespace($(this).prev('.query-namespace').text());
+ EDITORS.query.focus();
+}
+
function submitQuery(e) {
try {
e.preventDefault();
@@ -654,10 +669,39 @@
EDITORS.query.save();
// do nothing if query is empty
- if($('#query-box').val().trim() == '') {
+ var query = $('#query-box').val().trim();
+ if(query == '') {
return;
}
+ var queryExists = false;
+
+ // see if this query is already in the history
+ $('#query-history tbody tr').each(function(i, row) {
+ if($(row).find('.query')[0].innerText == query && $(row).find('.query-namespace').text() == NAMESPACE) {
+ // clear the old results and set the time to now
+ $(row).find('.query-time').text(new Date().toISOString());
+ $(row).find('.query-results').text('...');
+ // move it to the top
+ $(row).prependTo('#query-history tbody');
+ queryExists = true;
+ return false;
+ }
+ });
+
+ if(!queryExists) {
+ // add this query to the history
+ var row = $('<tr>').prependTo($('#query-history tbody'));
+ row.append('<td class="query-time">' + new Date().toISOString() + '</td>');
+ row.append('<td class="query-namespace">' + NAMESPACE + '</td>');
+ var cell = $('<td class="query">').appendTo(row);
+ cell.text(query);
+ cell.html(cell.html().replace('\n', '<br>'));
+ row.append('<td class="query-results">...</td>');
+ }
+
+ $('#query-history').show();
+
var url = RO_URL_PREFIX + 'namespace/' + NAMESPACE + '/sparql';
var settings = {
type: 'POST',
@@ -817,6 +861,10 @@
$('#download-link').remove();
}
+function updateResultCount(count) {
+ $('#query-history tbody tr:first td.query-results').text(count);
+}
+
function showQueryResults(data) {
$('#query-response').empty();
$('#query-export-rdf').hide();
@@ -846,6 +894,7 @@
table.append(tr);
}
}
+ updateResultCount(rows.length);
} else {
// JSON
// save data for export and pagination
@@ -854,6 +903,7 @@
if(typeof(data.boolean) != 'undefined') {
// ASK query
table.append('<tr><td>' + data.boolean + '</td></tr>').addClass('boolean');
+ updateResultCount('' + data.boolean);
return;
}
@@ -895,6 +945,7 @@
table.append(thead);
$('#total-results').html(data.results.bindings.length);
+ updateResultCount(data.results.bindings.length);
setNumberOfPages();
showPage(1);
@@ -1138,10 +1189,10 @@
} else {
var uri = col.value;
if(col.type == 'uri') {
- uri = '<' + uri + '>';
+ uri = abbreviate(uri);
}
}
- output = escapeHTML(uri).replace(/\n/g, '<br>');
+ var output = escapeHTML(uri).replace(/\n/g, '<br>');
if(col.type == 'uri' || col.type == 'sid') {
output = '<a href="' + buildExploreHash(uri) + '">' + output + '</a>';
}
@@ -1383,9 +1434,11 @@
}
function abbreviate(uri) {
- for(var ns in NAMESPACE_SHORTCUTS) {
- if(uri.indexOf(NAMESPACE_SHORTCUTS[ns]) == 0) {
- return uri.replace(NAMESPACE_SHORTCUTS[ns], ns + ':');
+ for(var nsGroup in NAMESPACE_SHORTCUTS) {
+ for(var ns in NAMESPACE_SHORTCUTS[nsGroup]) {
+ if(uri.indexOf(NAMESPACE_SHORTCUTS[nsGroup][ns]) == 0) {
+ return uri.replace(NAMESPACE_SHORTCUTS[nsGroup][ns], ns + ':');
+ }
}
}
return '<' + uri + '>';
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-06-26 22:48:06
|
Revision: 8505
http://sourceforge.net/p/bigdata/code/8505
Author: tobycraig
Date: 2014-06-26 22:47:58 +0000 (Thu, 26 Jun 2014)
Log Message:
-----------
Increased CodeMirror editor font size
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css 2014-06-25 01:22:52 UTC (rev 8504)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css 2014-06-26 22:47:58 UTC (rev 8505)
@@ -212,6 +212,7 @@
.CodeMirror {
margin: 5px 0;
border: 1px solid #e1e1e1;
+ font-size: 125%;
}
.CodeMirror-placeholder {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-06-25 01:22:56
|
Revision: 8504
http://sourceforge.net/p/bigdata/code/8504
Author: tobycraig
Date: 2014-06-25 01:22:52 +0000 (Wed, 25 Jun 2014)
Log Message:
-----------
#975 - Added colour to health page
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css 2014-06-24 19:14:03 UTC (rev 8503)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/css/style.css 2014-06-25 01:22:52 UTC (rev 8504)
@@ -340,3 +340,15 @@
border: 1px solid #e1e1e1;
box-sizing: border-box;
}
+
+.health-good {
+ background-color: lightgreen;
+}
+
+.health-warning {
+ background-color: orange;
+}
+
+.health-bad {
+ background-color: red;
+}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-24 19:14:03 UTC (rev 8503)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-25 01:22:52 UTC (rev 8504)
@@ -1360,6 +1360,9 @@
var date = new Date(data[key]);
data[key] = date.toString();
}
+ if(key == 'status') {
+ $('#health-overview').removeClass('health-good health-warning health-bad').addClass('health-' + data[key].toLowerCase());
+ }
$('#health-' + key + ' span').html(data[key]);
}
})
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-06-24 19:14:10
|
Revision: 8503
http://sourceforge.net/p/bigdata/code/8503
Author: tobycraig
Date: 2014-06-24 19:14:03 +0000 (Tue, 24 Jun 2014)
Log Message:
-----------
#975 - Added simple health panel
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HAStatusServletUtil.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/StatusServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/IMimeTypes.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HAStatusServletUtil.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HAStatusServletUtil.java 2014-06-20 12:00:30 UTC (rev 8502)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HAStatusServletUtil.java 2014-06-24 19:14:03 UTC (rev 8503)
@@ -25,11 +25,13 @@
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
+import java.io.StringWriter;
import java.math.BigInteger;
import java.net.InetSocketAddress;
import java.security.DigestException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
+import java.util.Date;
import java.util.Iterator;
import java.util.UUID;
import java.util.concurrent.TimeoutException;
@@ -65,6 +67,8 @@
import com.bigdata.quorum.zk.ZKQuorumImpl;
import com.bigdata.rdf.sail.webapp.StatusServlet.DigestEnum;
import com.bigdata.zookeeper.DumpZookeeper;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
/**
* Class supports the {@link StatusServlet} and isolates code that has a
@@ -885,6 +889,68 @@
}
+ /**
+ * Basic server health info
+ *
+ * @param req
+ * @param resp
+ * @throws TimeoutException
+ * @throws InterruptedException
+ * @throws AsynchronousQuorumCloseException
+ * @throws IOException
+ */
+ public void doHealthStatus(final HttpServletRequest req,
+ final HttpServletResponse resp) throws IOException {
+
+ if (!(indexManager instanceof HAJournal))
+ return;
+
+ final HAJournal journal = (HAJournal) indexManager;
+
+ final Quorum<HAGlue, QuorumService<HAGlue>> quorum = journal
+ .getQuorum();
+
+ StringWriter writer = new StringWriter();
+ JsonFactory factory = new JsonFactory();
+ JsonGenerator json = factory.createGenerator(writer);
+
+ json.writeStartObject();
+
+ json.writeFieldName("version");
+ json.writeString("1.0"); // FIXME
+ json.writeFieldName("timestamp");
+ json.writeNumber(new Date().getTime()); // FIXME
+ if(quorum.isQuorumFullyMet(quorum.token())) {
+ json.writeFieldName("status");
+ json.writeString("Good");
+ json.writeFieldName("details");
+ json.writeString("All servers joined");
+ } else {
+ // at least one server is not available, so status is either Warning or Bad
+ json.writeFieldName("status");
+ if(quorum.isQuorumMet()) {
+ json.writeString("Warning");
+ } else {
+ json.writeString("Bad");
+ }
+ json.writeFieldName("details");
+ json.writeString("Only " + quorum.getJoined().length + " of target " +
+ quorum.replicationFactor() + " servers joined");
+ }
+
+ json.writeEndObject();
+ json.close();
+
+ // TODO Alternatively "max-age=1" for max-age in seconds.
+ resp.addHeader("Cache-Control", "no-cache");
+
+ BigdataRDFServlet.buildResponse(resp, BigdataRDFServlet.HTTP_OK,
+ BigdataRDFServlet.MIME_APPLICATION_JSON, writer.toString());
+
+ return;
+
+ }
+
// /**
// * Impose a lexical ordering on the file names. This is used for the HALog
// * and snapshot file names. The main component of those file names is the
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/StatusServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/StatusServlet.java 2014-06-20 12:00:30 UTC (rev 8502)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/StatusServlet.java 2014-06-24 19:14:03 UTC (rev 8503)
@@ -187,6 +187,11 @@
static final String HA = "HA";
/**
+ * Request basic server health information.
+ */
+ static final String HEALTH = "health";
+
+ /**
* Handles CANCEL requests (terminate a running query).
*/
@Override
@@ -396,6 +401,14 @@
return;
}
+ if (req.getParameter(HEALTH) != null
+ && getIndexManager() instanceof AbstractJournal
+ && ((AbstractJournal) getIndexManager()).getQuorum() != null) { // for HA1
+ new HAStatusServletUtil(getIndexManager()).doHealthStatus(req, resp);
+
+ return;
+ }
+
// IRunningQuery objects currently running on the query controller.
final boolean showQueries = req.getParameter(SHOW_QUERIES) != null;
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/IMimeTypes.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/IMimeTypes.java 2014-06-20 12:00:30 UTC (rev 8502)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/IMimeTypes.java 2014-06-24 19:14:03 UTC (rev 8503)
@@ -37,6 +37,7 @@
MIME_DEFAULT_BINARY = "application/octet-stream",
MIME_APPLICATION_XML = "application/xml",
MIME_TEXT_JAVASCRIPT = "text/javascript",
+ MIME_APPLICATION_JSON = "application/json",
/**
* The traditional encoding of URL query parameters within a POST
* message body.
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-06-20 12:00:30 UTC (rev 8502)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-06-24 19:14:03 UTC (rev 8503)
@@ -25,6 +25,7 @@
<a data-target="update">Update</a>
<a data-target="explore">Explore</a>
<a data-target="status">Status</a>
+ <a data-target="health">Health</a>
<a data-target="performance">Performance</a>
<a data-target="namespaces">Namespaces</a>
<p>Current namespace: <span id="current-namespace"></span></p>
@@ -185,6 +186,18 @@
</div>
+ <div class="tab" id="health-tab">
+
+ <div class="box">
+ <p id="health-status">Status: <span></span></p>
+ <p id="health-details">Details: <span></span></p>
+ <p id="health-version">Version: <span></span></p>
+ <p id="health-timestamp">Timestamp: <span></span></p>
+ <p><a href="#" id="health-refresh">Refresh</a></p>
+ </div>
+
+ </div>
+
<div class="tab" id="performance-tab">
<div class="box"></div>
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-20 12:00:30 UTC (rev 8502)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-24 19:14:03 UTC (rev 8503)
@@ -1348,7 +1348,23 @@
});
}
+/* Health */
+$('#tab-selector a[data-target=health], #health-refresh').click(getHealth);
+
+function getHealth(e) {
+ e.preventDefault();
+ $.get('/status?health', function(data) {
+ for(var key in data) {
+ if(key == 'timestamp') {
+ var date = new Date(data[key]);
+ data[key] = date.toString();
+ }
+ $('#health-' + key + ' span').html(data[key]);
+ }
+ })
+}
+
/* Performance */
$('#tab-selector a[data-target=performance]').click(function(e) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2014-06-20 12:00:33
|
Revision: 8502
http://sourceforge.net/p/bigdata/code/8502
Author: martyncutcher
Date: 2014-06-20 12:00:30 +0000 (Fri, 20 Jun 2014)
Log Message:
-----------
Branch to develop and test a non-recycling constant allocator for the RWStore. The specific use-case is for out-of-line raw records for the BTree indices. The goal is to reduce both the storage overhead (by reducing slot waste) and runtime overhead (by reducing the amount of FixedAllocators).
Added Paths:
-----------
branches/RWSTORE_CONSTANT_STORE/
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-06-17 21:14:40
|
Revision: 8501
http://sourceforge.net/p/bigdata/code/8501
Author: thompsonbry
Date: 2014-06-17 21:14:25 +0000 (Tue, 17 Jun 2014)
Log Message:
-----------
Continued progress in support of #566 and #753. I am iterating over the REST API method implementations and restructuring them to support both #566 and, by extension, #753 as well. It is not necessary to change over to group commit in order to derive the benefit from this refactoring, but when we do change over it will be with a single boolean switch from the existing operational mode to the group commit operational mode.
I have modified the code to execute the RestApiTask in the caller's thread for the non-group-commit code path. This avoids the potential introduction of another thread for heavy query workloads.
I have cleaned up some of the launderThrowable() invocations to provide better information about the REST API request that failed. This is not yet systematic.
SPARQL Query and SPARQL UPDATE now go through the RestApiTask pattern.
Continuing to identify, document, and work through potential problems in the REST API that would conflict with group commit semantics.
All tests are passing. Group commit is still disabled.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/BigdataStatics.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BlueprintsServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/DeleteServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/DescribeCacheServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/MultiTenancyServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/RESTServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/RestApiTask.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/StatusServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/UpdateServlet.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/BigdataStatics.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/BigdataStatics.java 2014-06-17 20:51:10 UTC (rev 8500)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/BigdataStatics.java 2014-06-17 21:14:25 UTC (rev 8501)
@@ -125,7 +125,6 @@
* @see <a href="- http://sourceforge.net/apps/trac/bigdata/ticket/566" >
* Concurrent unisolated operations against multiple KBs </a>
*/
- public static final boolean NSS_GROUP_COMMIT = Boolean
- .getBoolean("com.bigdata.nssGroupCommit");
+ public static final boolean NSS_GROUP_COMMIT = Boolean.getBoolean("com.bigdata.nssGroupCommit");
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java 2014-06-17 20:51:10 UTC (rev 8500)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java 2014-06-17 21:14:25 UTC (rev 8501)
@@ -95,6 +95,7 @@
import com.bigdata.rdf.sail.ISPARQLUpdateListener;
import com.bigdata.rdf.sail.SPARQLUpdateEvent;
import com.bigdata.rdf.sail.sparql.Bigdata2ASTSPARQLParser;
+import com.bigdata.rdf.sail.webapp.RestApiTask.RestApiMutationTask;
import com.bigdata.rdf.sail.webapp.client.StringUtil;
import com.bigdata.rdf.sparql.ast.ASTContainer;
import com.bigdata.rdf.sparql.ast.QueryHints;
@@ -365,13 +366,10 @@
/**
* Immediate shutdown interrupts any running queries.
*
- * FIXME Must abort any open transactions. This does not matter for the
- * standalone database, but it will make a difference in scale-out. The
- * transaction identifiers could be obtained from the {@link #queries} map.
- *
- * FIXME This must also abort any running updates. Those are currently
- * running in thread handling the {@link HttpServletRequest}, however it
- * probably makes sense to execute them on a bounded thread pool as well.
+ * FIXME GROUP COMMIT: Shutdown should abort open transactions (including
+ * queries and updates). This hould be addressed when we handle group commit
+ * since that provides us with a means to recognize and interrupt each
+ * running {@link RestApiTask}.
*/
void shutdownNow() {
@@ -1135,83 +1133,125 @@
abstract protected void doQuery(BigdataSailRepositoryConnection cxn,
OutputStream os) throws Exception;
- @Override
- final public Void call() throws Exception {
- BigdataSailRepositoryConnection cxn = null;
- boolean success = false;
- try {
- // Note: Will be UPDATE connection if UPDATE request!!!
- cxn = getQueryConnection(namespace, timestamp);
- if(log.isTraceEnabled())
- log.trace("Query running...");
- beginNanos = System.nanoTime();
- if (explain && !update) {
- /*
- * The data goes to a bit bucket and we send an
- * "explanation" of the query evaluation back to the caller.
- *
- * Note: The trick is how to get hold of the IRunningQuery
- * object. It is created deep within the Sail when we
- * finally submit a query plan to the query engine. We have
- * the queryId (on queryId2), so we can look up the
- * IRunningQuery in [m_queries] while it is running, but
- * once it is terminated the IRunningQuery will have been
- * cleared from the internal map maintained by the
- * QueryEngine, at which point we can not longer find it.
- *
- * Note: We can't do this for UPDATE since it would have a
- * side-effect anyway. The way to "EXPLAIN" an UPDATE is to
- * break it down into the component QUERY bits and execute
- * those.
- */
- doQuery(cxn, new NullOutputStream());
- success = true;
- } else {
- doQuery(cxn, os);
- success = true;
- os.flush();
- os.close();
- }
- if (log.isTraceEnabled())
- log.trace("Query done.");
- return null;
- } finally {
- endNanos = System.nanoTime();
- m_queries.remove(queryId);
- if (queryId2 != null) m_queries2.remove(queryId2);
-// if (os != null) {
-// try {
-// os.close();
-// } catch (Throwable t) {
-// log.error(t, t);
-// }
-// }
- if (cxn != null) {
- if (!success && !cxn.isReadOnly()) {
+ /**
+ * Task for executing a SPARQL QUERY or SPARQL UPDATE.
+ * <p>
+ * See {@link AbstractQueryTask#update} to decide whether this task is a
+ * QUERY or an UPDATE.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan
+ * Thompson</a>
+ */
+ private class SparqlRestApiTask extends RestApiTask<Void> {
+
+ public SparqlRestApiTask(final HttpServletRequest req,
+ final HttpServletResponse resp, final String namespace,
+ final long timestamp) {
+
+ super(req, resp, namespace, timestamp);
+
+ }
+
+ @Override
+ public Void call() throws Exception {
+ BigdataSailRepositoryConnection cxn = null;
+ boolean success = false;
+ try {
+ // Note: Will be UPDATE connection if UPDATE request!!!
+ cxn = getQueryConnection();//namespace, timestamp);
+ if(log.isTraceEnabled())
+ log.trace("Query running...");
+ beginNanos = System.nanoTime();
+ if (explain && !update) {
/*
- * Force rollback of the connection.
+ * The data goes to a bit bucket and we send an
+ * "explanation" of the query evaluation back to the caller.
*
- * Note: It is possible that the commit has already been
- * processed, in which case this rollback() will be a
- * NOP. This can happen when there is an IO error when
- * communicating with the client, but the database has
- * already gone through a commit.
+ * Note: The trick is how to get hold of the IRunningQuery
+ * object. It is created deep within the Sail when we
+ * finally submit a query plan to the query engine. We have
+ * the queryId (on queryId2), so we can look up the
+ * IRunningQuery in [m_queries] while it is running, but
+ * once it is terminated the IRunningQuery will have been
+ * cleared from the internal map maintained by the
+ * QueryEngine, at which point we can not longer find it.
+ *
+ * Note: We can't do this for UPDATE since it would have a
+ * side-effect anyway. The way to "EXPLAIN" an UPDATE is to
+ * break it down into the component QUERY bits and execute
+ * those.
*/
+ doQuery(cxn, new NullOutputStream());
+ success = true;
+ } else {
+ doQuery(cxn, os);
+ success = true;
+ os.flush();
+ os.close();
+ }
+ if (log.isTraceEnabled())
+ log.trace("Query done.");
+ return null;
+ } finally {
+ endNanos = System.nanoTime();
+ m_queries.remove(queryId);
+ if (queryId2 != null) m_queries2.remove(queryId2);
+// if (os != null) {
+// try {
+// os.close();
+// } catch (Throwable t) {
+// log.error(t, t);
+// }
+// }
+ if (cxn != null) {
+ if (!success && !cxn.isReadOnly()) {
+ /*
+ * Force rollback of the connection.
+ *
+ * Note: It is possible that the commit has already been
+ * processed, in which case this rollback() will be a
+ * NOP. This can happen when there is an IO error when
+ * communicating with the client, but the database has
+ * already gone through a commit.
+ */
+ try {
+ // Force rollback of the connection.
+ cxn.rollback();
+ } catch (Throwable t) {
+ log.error(t, t);
+ }
+ }
try {
- // Force rollback of the connection.
- cxn.rollback();
+ // Force close of the connection.
+ cxn.close();
} catch (Throwable t) {
log.error(t, t);
}
}
- try {
- // Force close of the connection.
- cxn.close();
- } catch (Throwable t) {
- log.error(t, t);
- }
}
}
+
+ }
+
+ @Override
+ final public Void call() throws Exception {
+
+ final String queryOrUpdateStr = astContainer.getQueryString();
+
+ try {
+
+ return BigdataServlet.submitApiTask(getIndexManager(),
+ new SparqlRestApiTask(req, resp, namespace, timestamp))
+ .get();
+
+ } catch (Throwable t) {
+
+ // FIXME GROUP_COMMIT: check calling stack for existing launderThrowable.
+ throw BigdataRDFServlet.launderThrowable(t, resp,
+ queryOrUpdateStr);
+
+ }
+
} // call()
} // class AbstractQueryTask
@@ -1234,6 +1274,7 @@
}
+ @Override
protected void doQuery(final BigdataSailRepositoryConnection cxn,
final OutputStream os) throws Exception {
@@ -2111,64 +2152,64 @@
}
- /**
- * Return a connection transaction, which may be read-only or support
- * update. When the timestamp is associated with a historical commit point,
- * this will be a read-only connection. When it is associated with the
- * {@link ITx#UNISOLATED} view or a read-write transaction, this will be a
- * mutable connection.
- *
- * @param namespace
- * The namespace.
- * @param timestamp
- * The timestamp.
- *
- * @throws RepositoryException
- */
- public BigdataSailRepositoryConnection getQueryConnection(
- final String namespace, final long timestamp)
- throws RepositoryException {
+// /**
+// * Return a connection transaction, which may be read-only or support
+// * update. When the timestamp is associated with a historical commit point,
+// * this will be a read-only connection. When it is associated with the
+// * {@link ITx#UNISOLATED} view or a read-write transaction, this will be a
+// * mutable connection.
+// *
+// * @param namespace
+// * The namespace.
+// * @param timestamp
+// * The timestamp.
+// *
+// * @throws RepositoryException
+// */
+// public BigdataSailRepositoryConnection getQueryConnection(
+// final String namespace, final long timestamp)
+// throws RepositoryException {
+//
+// /*
+// * Note: [timestamp] will be a read-only tx view of the triple store if
+// * a READ_LOCK was specified when the NanoSparqlServer was started
+// * (unless the query explicitly overrides the timestamp of the view on
+// * which it will operate).
+// */
+// final AbstractTripleStore tripleStore = getTripleStore(namespace,
+// timestamp);
+//
+// if (tripleStore == null) {
+//
+// throw new DatasetNotFoundException("Not found: namespace="
+// + namespace + ", timestamp="
+// + TimestampUtility.toString(timestamp));
+//
+// }
+//
+// // Wrap with SAIL.
+// final BigdataSail sail = new BigdataSail(tripleStore);
+//
+// final BigdataSailRepository repo = new BigdataSailRepository(sail);
+//
+// repo.initialize();
+//
+// if (TimestampUtility.isReadOnly(timestamp)) {
+//
+// return (BigdataSailRepositoryConnection) repo
+// .getReadOnlyConnection(timestamp);
+//
+// }
+//
+// // Read-write connection.
+// final BigdataSailRepositoryConnection conn = repo.getConnection();
+//
+// conn.setAutoCommit(false);
+//
+// return conn;
+//
+// }
- /*
- * Note: [timestamp] will be a read-only tx view of the triple store if
- * a READ_LOCK was specified when the NanoSparqlServer was started
- * (unless the query explicitly overrides the timestamp of the view on
- * which it will operate).
- */
- final AbstractTripleStore tripleStore = getTripleStore(namespace,
- timestamp);
-
- if (tripleStore == null) {
-
- throw new DatasetNotFoundException("Not found: namespace="
- + namespace + ", timestamp="
- + TimestampUtility.toString(timestamp));
-
- }
-
- // Wrap with SAIL.
- final BigdataSail sail = new BigdataSail(tripleStore);
-
- final BigdataSailRepository repo = new BigdataSailRepository(sail);
-
- repo.initialize();
-
- if (TimestampUtility.isReadOnly(timestamp)) {
-
- return (BigdataSailRepositoryConnection) repo
- .getReadOnlyConnection(timestamp);
-
- }
-
- // Read-write connection.
- final BigdataSailRepositoryConnection conn = repo.getConnection();
-
- conn.setAutoCommit(false);
-
- return conn;
-
- }
-
/**
* Return a read-only view of the {@link AbstractTripleStore} for the given
* namespace will read from the commit point associated with the given
@@ -2182,12 +2223,17 @@
* @return The {@link AbstractTripleStore} -or- <code>null</code> if none is
* found for that namespace and timestamp.
*
- * @todo enforce historical query by making sure timestamps conform (we do
- * not want to allow read/write tx queries unless update semantics are
- * introduced ala SPARQL 1.1).
- *
- * @todo Use a distributed read-only tx for queries (it would be nice if a
- * tx used 2PL to specify which namespaces it could touch).
+ * FIXME GROUP_COMMIT: Review all callers. They are suspect. The
+ * code will sometimes resolve the KB as of the timestamp, but,
+ * given that the default is to read against the lastCommitTime,
+ * that does NOT prevent a concurrent destroy or create of a KB that
+ * invalidates such a pre-condition test. The main reason for such
+ * pre-condition tests is to provide nice HTTP status code responses
+ * when an identified namespace does (or does not) exist. The better
+ * way to handle this is by pushing the pre-condition test down into
+ * the {@link RestApiTask} and then throwning out an appropriate
+ * marked exception that gets correctly converted into an HTTP
+ * BAD_REQUEST message rather than sending back a stack trace.
*/
public AbstractTripleStore getTripleStore(final String namespace,
final long timestamp) {
@@ -2214,8 +2260,12 @@
* @throws SailException
*
* @throws RepositoryException
+ *
+ * FIXME GROUP COMMIT: This is deprecated by the support for
+ * {@link RestApiMutationTask}s
*/
- public BigdataSailRepositoryConnection getUnisolatedConnection( // FIXME REVIEW CALLERS
+ @Deprecated // deprecated by the
+ BigdataSailRepositoryConnection getUnisolatedConnection(
final String namespace) throws SailException, RepositoryException {
// resolve the default namespace.
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFServlet.java 2014-06-17 20:51:10 UTC (rev 8500)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFServlet.java 2014-06-17 21:14:25 UTC (rev 8501)
@@ -47,8 +47,6 @@
import org.openrdf.model.Resource;
import org.openrdf.model.Statement;
import org.openrdf.model.impl.URIImpl;
-import org.openrdf.repository.RepositoryException;
-import org.openrdf.repository.RepositoryResult;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFHandlerException;
import org.openrdf.rio.RDFWriter;
@@ -59,7 +57,6 @@
import com.bigdata.rdf.properties.PropertiesWriter;
import com.bigdata.rdf.properties.PropertiesWriterRegistry;
import com.bigdata.rdf.rules.ConstraintViolationException;
-import com.bigdata.rdf.sail.webapp.XMLBuilder.Node;
import com.bigdata.util.InnerCause;
/**
@@ -398,41 +395,7 @@
buildResponse(resp, HTTP_OK, MIME_APPLICATION_XML, w.toString());
}
-
- /**
- * Report the contexts back to the user agent.
- *
- * @param resp
- * The response.
- * @param it
- * The iteration of contexts.
- * @param elapsed
- * The elapsed time (milliseconds).
- *
- * @throws IOException
- */
- static protected void reportContexts(final HttpServletResponse resp,
- final RepositoryResult<Resource> contexts, final long elapsed)
- throws IOException, RepositoryException {
-
- final StringWriter w = new StringWriter();
- final XMLBuilder t = new XMLBuilder(w);
-
- final Node root = t.root("contexts");
-
- while (contexts.hasNext()) {
-
- root.node("context").attr("uri", contexts.next()).close();
-
- }
-
- root.close();
-
- buildResponse(resp, HTTP_OK, MIME_APPLICATION_XML, w.toString());
-
- }
-
/**
* Send an RDF Graph as a response using content negotiation.
*
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataServlet.java 2014-06-17 20:51:10 UTC (rev 8500)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataServlet.java 2014-06-17 21:14:25 UTC (rev 8501)
@@ -34,6 +34,7 @@
import java.util.List;
import java.util.Set;
import java.util.concurrent.Future;
+import java.util.concurrent.FutureTask;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServlet;
@@ -53,6 +54,7 @@
import com.bigdata.rdf.sail.webapp.client.IMimeTypes;
import com.bigdata.rdf.sail.webapp.lbs.IHALoadBalancerPolicy;
import com.bigdata.rdf.store.AbstractTripleStore;
+import com.bigdata.resources.IndexManager;
import com.bigdata.service.IBigdataFederation;
/**
@@ -216,22 +218,49 @@
* @see <a href="- http://sourceforge.net/apps/trac/bigdata/ticket/566" >
* Concurrent unisolated operations against multiple KBs </a>
*/
- @SuppressWarnings({ "unchecked", "rawtypes" })
protected <T> Future<T> submitApiTask(final RestApiTask<T> task)
throws DatasetNotFoundException {
+ final IIndexManager indexManager = getIndexManager();
+
+ return submitApiTask(indexManager, task);
+
+ }
+
+ /**
+ * Submit a task and return a {@link Future} for that task. The task will be
+ * run on the appropriate executor service depending on the nature of the
+ * backing database and the view required by the task.
+ *
+ * @param indexManager
+ * The {@link IndexManager}.
+ * @param task
+ * The task.
+ *
+ * @return The {@link Future} for that task.
+ *
+ * @throws DatasetNotFoundException
+ *
+ * @see <a href="http://sourceforge.net/apps/trac/bigdata/ticket/753" > HA
+ * doLocalAbort() should interrupt NSS requests and AbstractTasks </a>
+ * @see <a href="- http://sourceforge.net/apps/trac/bigdata/ticket/566" >
+ * Concurrent unisolated operations against multiple KBs </a>
+ */
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ static protected <T> Future<T> submitApiTask(
+ final IIndexManager indexManager, final RestApiTask<T> task)
+ throws DatasetNotFoundException {
+
final String namespace = task.getNamespace();
final long timestamp = task.getTimestamp();
- final IIndexManager indexManager = getIndexManager();
-
if (!BigdataStatics.NSS_GROUP_COMMIT || indexManager instanceof IBigdataFederation
|| TimestampUtility.isReadOnly(timestamp)
) {
/*
- * Run on a normal executor service.
+ * Execute the REST API task.
*
* Note: For scale-out, the operation will be applied using
* client-side global views of the indices.
@@ -240,10 +269,31 @@
* a Journal). This is helpful since we can avoid some overhead
* associated the AbstractTask lock declarations.
*/
-
- return indexManager.getExecutorService().submit(
+ // Wrap Callable.
+ final FutureTask<T> ft = new FutureTask<T>(
new RestApiTaskForIndexManager(indexManager, task));
+ if (true) {
+
+ /*
+ * Caller runs (synchronous execution)
+ *
+ * Note: By having the caller run the task here we avoid
+ * consuming another thread.
+ */
+ ft.run();
+
+ } else {
+
+ /*
+ * Run on a normal executor service.
+ */
+ indexManager.getExecutorService().submit(ft);
+
+ }
+
+ return ft;
+
} else {
/**
@@ -282,7 +332,7 @@
}
}
-
+
/**
* Acquire the locks for the named indices associated with the specified KB.
*
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BlueprintsServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BlueprintsServlet.java 2014-06-17 20:51:10 UTC (rev 8500)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BlueprintsServlet.java 2014-06-17 21:14:25 UTC (rev 8501)
@@ -33,8 +33,8 @@
import com.bigdata.blueprints.BigdataGraphBulkLoad;
import com.bigdata.rdf.sail.BigdataSailRepositoryConnection;
+import com.bigdata.rdf.sail.webapp.RestApiTask.RestApiMutationTask;
import com.bigdata.rdf.sail.webapp.client.MiniMime;
-import com.bigdata.rdf.store.AbstractTripleStore;
import com.tinkerpop.blueprints.util.io.graphml.GraphMLReader;
/**
@@ -69,16 +69,8 @@
protected void doPost(final HttpServletRequest req,
final HttpServletResponse resp) throws IOException {
- final long begin = System.currentTimeMillis();
-
- final String namespace = getNamespace(req);
-
- final long timestamp = getTimestamp(req);
-
- final AbstractTripleStore tripleStore = getBigdataRDFContext()
- .getTripleStore(namespace, timestamp);
-
- if (tripleStore == null) {
+ if (getBigdataRDFContext().getTripleStore(getNamespace(req),
+ getTimestamp(req)) == null) {
/*
* There is no such triple/quad store instance.
*/
@@ -104,12 +96,37 @@
try {
+ submitApiTask(
+ new BlueprintsPostTask(req, resp, getNamespace(req),
+ getTimestamp(req))).get();
+
+ } catch (Throwable t) {
+
+ throw BigdataRDFServlet.launderThrowable(t, resp, "");
+
+ }
+
+ }
+
+ private static class BlueprintsPostTask extends RestApiMutationTask<Void> {
+
+ public BlueprintsPostTask(HttpServletRequest req,
+ HttpServletResponse resp, String namespace, long timestamp) {
+
+ super(req, resp, namespace, timestamp);
+
+ }
+
+ @Override
+ public Void call() throws Exception {
+
+ final long begin = System.currentTimeMillis();
+
BigdataSailRepositoryConnection conn = null;
boolean success = false;
try {
- conn = getBigdataRDFContext()
- .getUnisolatedConnection(namespace);
+ conn = getUnisolatedConnection();
final BigdataGraphBulkLoad graph = new BigdataGraphBulkLoad(conn);
@@ -123,10 +140,11 @@
final long elapsed = System.currentTimeMillis() - begin;
- reportModifiedCount(resp, nmodified, elapsed);
-
- return;
+ reportModifiedCount(nmodified, elapsed);
+ // Done.
+ return null;
+
} finally {
if (conn != null) {
@@ -137,15 +155,11 @@
conn.close();
}
-
+
}
- } catch (Throwable t) {
+ }
- throw BigdataRDFServlet.launderThrowable(t, resp, "");
-
- }
-
}
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/DeleteServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/DeleteServlet.java 2014-06-17 20:51:10 UTC (rev 8500)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/DeleteServlet.java 2014-06-17 21:14:25 UTC (rev 8501)
@@ -25,6 +25,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.PipedOutputStream;
+import java.util.Arrays;
import java.util.concurrent.FutureTask;
import java.util.concurrent.atomic.AtomicLong;
@@ -270,8 +271,6 @@
final HttpServletResponse resp) throws IOException {
final String baseURI = req.getRequestURL().toString();
-
- final String namespace = getNamespace(req);
final String contentType = req.getContentType();
@@ -281,68 +280,70 @@
if (log.isInfoEnabled())
log.info("Request body: " + contentType);
- try {
+ /**
+ * There is a request body, so let's try and parse it.
+ *
+ * <a href="https://sourceforge.net/apps/trac/bigdata/ticket/620">
+ * UpdateServlet fails to parse MIMEType when doing conneg. </a>
+ */
- /**
- * There is a request body, so let's try and parse it.
- *
- * <a href="https://sourceforge.net/apps/trac/bigdata/ticket/620">
- * UpdateServlet fails to parse MIMEType when doing conneg. </a>
- */
+ final RDFFormat format = RDFFormat.forMIMEType(new MiniMime(
+ contentType).getMimeType());
- final RDFFormat format = RDFFormat.forMIMEType(new MiniMime(
- contentType).getMimeType());
+ if (format == null) {
- if (format == null) {
+ buildResponse(resp, HTTP_BADREQUEST, MIME_TEXT_PLAIN,
+ "Content-Type not recognized as RDF: " + contentType);
- buildResponse(resp, HTTP_BADREQUEST, MIME_TEXT_PLAIN,
- "Content-Type not recognized as RDF: " + contentType);
+ return;
- return;
+ }
- }
+ final RDFParserFactory rdfParserFactory = RDFParserRegistry
+ .getInstance().get(format);
- final RDFParserFactory rdfParserFactory = RDFParserRegistry
- .getInstance().get(format);
+ if (rdfParserFactory == null) {
- if (rdfParserFactory == null) {
+ buildResponse(resp, HTTP_INTERNALERROR, MIME_TEXT_PLAIN,
+ "Parser factory not found: Content-Type=" + contentType
+ + ", format=" + format);
- buildResponse(resp, HTTP_INTERNALERROR, MIME_TEXT_PLAIN,
- "Parser factory not found: Content-Type=" + contentType
- + ", format=" + format);
+ return;
- return;
+ }
+ /*
+ * Allow the caller to specify the default contexts.
+ */
+ final Resource[] defaultContext;
+ {
+ final String[] s = req.getParameterValues("context-uri");
+ if (s != null && s.length > 0) {
+ try {
+ defaultContext = toURIs(s);
+ } catch (IllegalArgumentException ex) {
+ buildResponse(resp, HTTP_INTERNALERROR, MIME_TEXT_PLAIN,
+ ex.getLocalizedMessage());
+ return;
+ }
+ } else {
+ defaultContext = new Resource[0];
}
+ }
- /*
- * Allow the caller to specify the default contexts.
- */
- final Resource[] defaultContext;
- {
- final String[] s = req.getParameterValues("context-uri");
- if (s != null && s.length > 0) {
- try {
- defaultContext = toURIs(s);
- } catch (IllegalArgumentException ex) {
- buildResponse(resp, HTTP_INTERNALERROR, MIME_TEXT_PLAIN,
- ex.getLocalizedMessage());
- return;
- }
- } else {
- defaultContext = new Resource[0];
- }
- }
-
+ try {
+
submitApiTask(
- new DeleteWithBodyTask(req, resp, namespace,
+ new DeleteWithBodyTask(req, resp, getNamespace(req),
ITx.UNISOLATED, baseURI, defaultContext,
rdfParserFactory)).get();
-
+
} catch (Throwable t) {
- throw BigdataRDFServlet.launderThrowable(t, resp, "");
-
+ throw BigdataRDFServlet.launderThrowable(t, resp,
+ "DELETE-WITH-BODY: baseURI=" + baseURI + ", context-uri="
+ + Arrays.toString(defaultContext));
+
}
}
@@ -523,7 +524,7 @@
}
if (log.isInfoEnabled())
- log.info("delete with access path: (s=" + s + ", p=" + p + ", o="
+ log.info("DELETE-WITH-ACCESS-PATH: (s=" + s + ", p=" + p + ", o="
+ o + ", c=" + c + ")");
try {
@@ -534,8 +535,9 @@
} catch (Throwable t) {
- throw BigdataRDFServlet.launderThrowable(t, resp, "s=" + s + ",p="
- + p + ",o=" + o + ",c=" + c);
+ throw BigdataRDFServlet.launderThrowable(t, resp,
+ "DELETE-WITH-ACCESS-PATH: (s=" + s + ",p=" + p + ",o=" + o
+ + ",c=" + c + ")");
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/DescribeCacheServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/DescribeCacheServlet.java 2014-06-17 20:51:10 UTC (rev 8500)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/DescribeCacheServlet.java 2014-06-17 21:14:25 UTC (rev 8501)
@@ -129,6 +129,10 @@
/**
* GET returns the DESCRIBE of the resource.
+ *
+ * FIXME DESCRIBE: TX ISOLATION for request but ensure that cache is not
+ * negatively effected by that isolation (i.e., how does the cache index
+ * based on time tx view).
*/
@Override
protected void doGet(final HttpServletRequest req,
@@ -369,14 +373,10 @@
os.flush();
} catch (Throwable e) {
-// try {
- throw BigdataRDFServlet.launderThrowable(e, resp,
- "DESCRIBE"
- // queryStr // TODO Report as "DESCRIBE uri(s)".
- );
-// } catch (Exception e1) {
-// throw new RuntimeException(e);
-// }
+
+ throw BigdataRDFServlet.launderThrowable(e, resp,
+ "DESCRIBE: uris=" + internalURIs);
+
}
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java 2014-06-17 20:51:10 UTC (rev 8500)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java 2014-06-17 21:14:25 UTC (rev 8501)
@@ -138,8 +138,6 @@
final String baseURI = req.getRequestURL().toString();
- final String namespace = getNamespace(req);
-
final String contentType = req.getContentType();
if (contentType == null)
@@ -175,32 +173,13 @@
if (rdfParserFactory == null) {
buildResponse(resp, HTTP_INTERNALERROR, MIME_TEXT_PLAIN,
- "Parser factory not found: Content-Type="
- + contentType + ", format=" + format);
-
- return;
+ "Parser factory not found: Content-Type=" + contentType
+ + ", format=" + format);
+ return;
+
}
-// /*
-// * Allow the caller to specify the default context.
-// */
-// final Resource defaultContext;
-// {
-// final String s = req.getParameter("context-uri");
-// if (s != null) {
-// try {
-// defaultContext = new URIImpl(s);
-// } catch (IllegalArgumentException ex) {
-// buildResponse(resp, HTTP_INTERNALERROR, MIME_TEXT_PLAIN,
-// ex.getLocalizedMessage());
-// return;
-// }
-// } else {
-// defaultContext = null;
-// }
-// }
-
/*
* Allow the caller to specify the default contexts.
*/
@@ -223,13 +202,16 @@
try {
submitApiTask(
- new InsertWithBodyTask(req, resp, namespace, ITx.UNISOLATED,
- baseURI, defaultContext, rdfParserFactory)).get();
+ new InsertWithBodyTask(req, resp, getNamespace(req),
+ ITx.UNISOLATED, baseURI, defaultContext,
+ rdfParserFactory)).get();
} catch (Throwable t) {
- throw BigdataRDFServlet.launderThrowable(t, resp, "");
-
+ throw BigdataRDFServlet.launderThrowable(t, resp,
+ "INSERT-WITH-BODY: baseURI=" + baseURI + ", context-uri="
+ + Arrays.toString(defaultContext));
+
}
}
@@ -385,25 +367,6 @@
}
-// /*
-// * Allow the caller to specify the default context.
-// */
-// final Resource defaultContext;
-// {
-// final String s = req.getParameter("context-uri");
-// if (s != null) {
-// try {
-// defaultContext = new URIImpl(s);
-// } catch (IllegalArgumentException ex) {
-// buildResponse(resp, HTTP_INTERNALERROR, MIME_TEXT_PLAIN,
-// ex.getLocalizedMessage());
-// return;
-// }
-// } else {
-// defaultContext = null;
-// }
-// }
-
/*
* Allow the caller to specify the default contexts.
*/
@@ -431,8 +394,9 @@
} catch (Throwable t) {
- throw launderThrowable(t, resp, "urls=" + urls);
-
+ throw launderThrowable(t, resp, "uri=" + urls + ", context-uri="
+ + Arrays.toString(defaultContext));
+
}
}
@@ -688,10 +652,10 @@
}
if (c.length >= 2) {
- // added to more than one context
- nmodified.addAndGet(c.length);
+ // added to more than one context
+ nmodified.addAndGet(c.length);
} else {
- nmodified.incrementAndGet();
+ nmodified.incrementAndGet();
}
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/MultiTenancyServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/MultiTenancyServlet.java 2014-06-17 20:51:10 UTC (rev 8500)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/MultiTenancyServlet.java 2014-06-17 21:14:25 UTC (rev 8501)
@@ -63,6 +63,10 @@
* NanoSparqlServer Admin API for Multi-tenant deployments</a>
*
* @author thompsonbry
+ *
+ * FIXME GROUP COMMIT: The CREATE and DESTROY operations require special
+ * attention. The other operations in this class also should use the new
+ * REST API pattern, but are not intrinsically sensitive.
*/
public class MultiTenancyServlet extends BigdataRDFServlet {
@@ -540,13 +544,6 @@
final long timestamp = getTimestamp(req);
-// if (timestamp == ITx.READ_COMMITTED) {
-//
-// // Use the last commit point.
-// timestamp = getIndexManager().getLastCommitTime();
-//
-// }
-
final long tx = getBigdataRDFContext().newTx(timestamp);
try {
@@ -582,13 +579,6 @@
final HttpServletResponse resp) throws IOException {
final long timestamp = getTimestamp(req);
-
-// if (timestamp == ITx.READ_COMMITTED) {
-//
-// // Use the last commit point.
-// timestamp = getIndexManager().getLastCommitTime();
-//
-// }
final boolean describeEachNamedGraph;
{
@@ -683,9 +673,7 @@
final VoID v = new VoID(g, tripleStore, serviceURI, aDataSet);
- v.describeDataSet(false/* describeStatistics */,
-// getBigdataRDFContext().getConfig().describeEachNamedGraph);
- describeEachNamedGraph);
+ v.describeDataSet(false/* describeStatistics */, describeEachNamedGraph);
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java 2014-06-17 20:51:10 UTC (rev 8500)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java 2014-06-17 21:14:25 UTC (rev 8501)
@@ -69,6 +69,8 @@
import com.bigdata.rdf.sail.webapp.BigdataRDFContext.AbstractQueryTask;
import com.bigdata.rdf.sail.webapp.BigdataRDFContext.RunningQuery;
import com.bigdata.rdf.sail.webapp.BigdataRDFContext.UpdateTask;
+import com.bigdata.rdf.sail.webapp.RestApiTask.RestApiQueryTask;
+import com.bigdata.rdf.sail.webapp.XMLBuilder.Node;
import com.bigdata.rdf.sail.webapp.client.EncodeDecodeValue;
import com.bigdata.rdf.sparql.ast.ASTContainer;
import com.bigdata.rdf.sparql.ast.QueryRoot;
@@ -163,11 +165,11 @@
if (req.getParameter(ATTR_UPDATE) != null) {
// SPARQL 1.1 UPDATE.
- doUpdate(req, resp);
+ doSparqlUpdate(req, resp);
} else if (RESTServlet.hasMimeType(req, MIME_SPARQL_UPDATE)) {
// SPARQL 1.1 UPDATE, see trac 711 for bug report motivating this case
- doUpdate(req, resp);
+ doSparqlUpdate(req, resp);
} else if (req.getParameter(ATTR_UUID) != null) {
@@ -187,7 +189,7 @@
} else {
// SPARQL Query.
- doQuery(req, resp);
+ doSparqlQuery(req, resp);
}
@@ -202,7 +204,7 @@
if (req.getParameter(ATTR_QUERY) != null) {
- doQuery(req, resp);
+ doSparqlQuery(req, resp);
} else if (req.getParameter(ATTR_UUID) != null) {
@@ -318,7 +320,7 @@
* @param resp
* @throws IOException
*/
- private void doUpdate(final HttpServletRequest req,
+ private void doSparqlUpdate(final HttpServletRequest req,
final HttpServletResponse resp) throws IOException {
if (!isWritable(getServletContext(), req, resp)) {
@@ -409,43 +411,9 @@
}
/**
- * FIXME GROUP COMMIT: We need to refactor the code that manages the
- * running queries in BigdataRDFServlet so we can separate out the
- * concurrency control of the views from the control over the #of
- * running queries and/or update requests and the metadata that we
- * manage to track and report on those requests.
- */
-// private static class SparqlUpdateTask extends RestApiMutationTask<Void> {
-//
-// /**
-// *
-// * @param namespace
-// * The namespace of the target KB instance.
-// * @param timestamp
-// * The timestamp used to obtain a mutable connection.
-// * @param baseURI
-// * The base URI for the operation.
-// */
-// public SparqlUpdateTask(final HttpServletRequest req,
-// final HttpServletResponse resp,
-// final String namespace, final long timestamp
-// ) {
-// super(req, resp, namespace, timestamp);
-// }
-//
-// @Override
-// public Void call() throws Exception {
-//
-//
-//
-// }
-//
-// }
-
- /**
* Run a SPARQL query.
*/
- void doQuery(final HttpServletRequest req, final HttpServletResponse resp)
+ void doSparqlQuery(final HttpServletRequest req, final HttpServletResponse resp)
throws IOException {
if (!isReadable(getServletContext(), req, resp)) {
@@ -1065,10 +1033,6 @@
return;
}
- final long begin = System.currentTimeMillis();
-
- final String namespace = getNamespace(req);
-
final Resource s;
final URI p;
final Value o;
@@ -1089,70 +1053,94 @@
+ o + ", c=" + c + ")");
try {
+
+ submitApiTask(
+ new EstCardTask(req, resp, getNamespace(req),
+ getTimestamp(req), //
+ s, p, o, c)).get();
- try {
+ } catch (Throwable t) {
- BigdataSailRepositoryConnection conn = null;
- try {
+ launderThrowable(t, resp, "ESTCARD: access path: (s=" + s + ", p="
+ + p + ", o=" + o + ", c=" + c + ")");
- final long timestamp = getTimestamp(req);
+ }
+
+ }
+
+ /**
+ * Helper task for the ESTCARD query.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ */
+ private static class EstCardTask extends RestApiQueryTask<Void> {
- conn = getBigdataRDFContext().getQueryConnection(
- namespace, timestamp);
+ private final Resource s;
+ private final URI p;
+ private final Value o;
+ private final Resource[] c;
+
+ public EstCardTask(final HttpServletRequest req,
+ final HttpServletResponse resp, final String namespace,
+ final long timestamp, final Resource s, final URI p,
+ final Value o, final Resource[] c) {
- // Range count all statements matching that access path.
- long rangeCount = 0;
- if (c != null && c.length > 0) {
- for (Resource r : c) {
- rangeCount += conn.getSailConnection()
- .getBigdataSail().getDatabase()
- .getAccessPath(s, p, o, r)
- .rangeCount(false/* exact */);
- }
- } else {
- rangeCount += conn.getSailConnection()
- .getBigdataSail().getDatabase()
- .getAccessPath(s, p, o, (Resource) null)
+ super(req, resp, namespace, timestamp);
+
+ this.s = s;
+ this.p = p;
+ this.o = o;
+ this.c = c;
+
+ }
+
+ @Override
+ public Void call() throws Exception {
+
+ final long begin = System.currentTimeMillis();
+
+ BigdataSailRepositoryConnection conn = null;
+ try {
+
+ conn = getQueryConnection();
+
+ // Range count all statements matching that access path.
+ long rangeCount = 0;
+ if (c != null && c.length > 0) {
+ for (Resource r : c) {
+ rangeCount += conn.getSailConnection().getBigdataSail()
+ .getDatabase().getAccessPath(s, p, o, r)
.rangeCount(false/* exact */);
}
-
- final long elapsed = System.currentTimeMillis() - begin;
-
- reportRangeCount(resp, rangeCount, elapsed);
+ } else {
+ rangeCount += conn.getSailConnection().getBigdataSail()
+ .getDatabase()
+ .getAccessPath(s, p, o, (Resource) null)
+ .rangeCount(false/* exact */);
+ }
- } catch(Throwable t) {
-
- if(conn != null)
- conn.rollback();
-
- throw new RuntimeException(t);
-
- } finally {
+ final long elapsed = System.currentTimeMillis() - begin;
- if (conn != null)
- conn.close();
+ reportRangeCount(resp, rangeCount, elapsed);
- }
+ return null;
- } catch (Throwable t) {
+ } finally {
- throw BigdataRDFServlet.launderThrowable(t, resp, "");
+ if (conn != null) {
- }
+ conn.close();
- } catch (Exception ex) {
+ }
- // Will be rendered as an INTERNAL_ERROR.
- throw new RuntimeException(ex);
+ }
}
- }
+ } // ESTCARD task.
/**
* Report on the contexts in use in the quads database.
- * @param req
- * @param resp
*/
private void doContexts(final HttpServletRequest req,
final HttpServletResponse resp) throws IOException {
@@ -1161,58 +1149,86 @@
// HA Quorum in use, but quorum is not met.
return;
}
-
- final long begin = System.currentTimeMillis();
-
- final String namespace = getNamespace(req);
try {
+
+ submitApiTask(
+ new GetContextsTask(req, resp, getNamespace(req),
+ getTimestamp(req))).get();
+ } catch (Throwable t) {
+
+ launderThrowable(t, resp, "GET-CONTEXTS");
+
+ }
+
+ }
+
+ /**
+ * Task to report the contexts used by a QUADS mode KB instance.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ */
+ private static class GetContextsTask extends RestApiQueryTask<Void> {
+
+ public GetContextsTask(final HttpServletRequest req,
+ final HttpServletResponse resp, final String namespace,
+ final long timestamp) {
+
+ super(req, resp, namespace, timestamp);
+
+ }
+
+ @Override
+ public Void call() throws Exception {
+
+ BigdataSailRepositoryConnection conn = null;
try {
- BigdataSailRepositoryConnection conn = null;
+ conn = getQueryConnection();
+
+ final StringWriter w = new StringWriter();
+
+ final RepositoryResult<Resource> it = conn.getContextIDs();
+
try {
- final long timestamp = getTimestamp(req);
+ final XMLBuilder t = new XMLBuilder(w);
- conn = getBigdataRDFContext().getQueryConnection(
- namespace, timestamp);
+ final Node root = t.root("contexts");
- final RepositoryResult<Resource> it = conn.getContextIDs();
-
- final long elapsed = System.currentTimeMillis() - begin;
-
- reportContexts(resp, it, elapsed);
+ while (it.hasNext()) {
- } catch(Throwable t) {
-
- if(conn != null)
- conn.rollback();
-
- throw new RuntimeException(t);
-
+ root.node("context").attr("uri", it.next()).close();
+
+ }
+
+ root.close();
+
} finally {
- if (conn != null)
- conn.close();
+ it.close();
}
- } catch (Throwable t) {
+ buildResponse(resp, HTTP_OK, MIME_APPLICATION_XML, w.toString());
+
+ return null;
- throw BigdataRDFServlet.launderThrowable(t, resp, "");
+ } finally {
- }
+ if (conn != null) {
- } catch (Exception ex) {
+ conn.close();
+
+ }
- // Will be rendered as an INTERNAL_ERROR.
- throw new RuntimeException(ex);
+ }
}
-
+
}
-
+
/**
* Private API reports the shards against which the access path would
* read.
@@ -1234,10 +1250,6 @@
return;
}
- final long begin = System.currentTimeMillis();
-
- final String namespace = getNamespace(req);
-
final boolean doRangeCount = true;
final Resource s;
final URI p;
@@ -1259,173 +1271,201 @@
+ o + ", c=" + c + ")");
try {
+
+ submitApiTask(
+ new ShardsTask(req, resp, getNamespace(req),
+ getTimestamp(req), s, p, o, c, doRangeCount)).get();
- try {
+ } catch (Throwable t) {
- BigdataSailRepositoryConnection conn = null;
- try {
+ launderThrowable(t, resp, "SHARDS: access path: (s=" + s + ", p="
+ + p + ", o=" + o + ", c=" + c + ")");
- final long timestamp = getTimestamp(req);
+ }
- conn = getBigdataRDFContext().getQueryConnection(
- namespace, timestamp);
+ }
- final AccessPath<?> accessPath = (AccessPath<?>) conn
- .getSailConnection().getBigdataSail().getDatabase()
- .getAccessPath(s, p, o, c);
-
- final ClientIndexView ndx = (ClientIndexView) accessPath
- .getIndex();
-
- final String charset = "utf-8";// TODO from request.
+ /**
+ * Task to report on the SHARDS used by a scale-out deployment.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ */
+ private static class ShardsTask extends RestApiQueryTask<Void> {
- resp.setContentType(BigdataServlet.MIME_TEXT_HTML);
- resp.setCharacterEncoding(charset);
- final Writer w = resp.getWriter();
- try {
+ private final Resource s;
+ private final URI p;
+ private final Value o;
+ private final Resource c;
+ private final boolean doRangeCount;
+
+ public ShardsTask(final HttpServletRequest req,
+ final HttpServletResponse resp, final String namespace,
+ final long timestamp, final Resource s, final URI p,
+ final Value o, final Resource c, final boolean doRangeCount) {
- final HTMLBuilder doc = new HTMLBuilder(charset, w);
-
- XMLBuilder.Node current = doc.root("html");
- {
- current = current.node("head");
- current.node("meta")
- .attr("http-equiv", "Content-Type")
- .attr("content",
- "text/html;charset=utf-8")
- .close();
- current.node("title")
- .textNoEncode("bigdata®").close();
- current = current.close();// close the head.
- }
+ super(req, resp, namespace, timestamp);
- // open the body
- current = current.node("body");
+ this.s = s;
+ this.p = p;
+ this.o = o;
+ this.c = c;
+ this.doRangeCount = doRangeCount;
+
+ }
- final IBigdataFederation<?> fed = (IBigdataFederation<?>) getBigdataRDFContext()
- .getIndexManager();
-
- final Iterator<PartitionLocator> itr = ndx.locatorScan(
- timestamp, accessPath.getFromKey(),
- accessPath.getToKey(), false/* reverseScan */);
+ @Override
+ public Void call() throws Exception {
- int nlocators = 0;
+ final long begin = System.currentTimeMillis();
+
+ BigdataSailRepositoryConnection conn = null;
+ try {
- // The distinct hosts on which the shards are located.
- final Map<String,AtomicInteger> hosts = new TreeMap<String,AtomicInteger>();
- ...
[truncated message content] |
|
From: <tho...@us...> - 2014-06-17 20:51:15
|
Revision: 8500
http://sourceforge.net/p/bigdata/code/8500
Author: thompsonbry
Date: 2014-06-17 20:51:10 +0000 (Tue, 17 Jun 2014)
Log Message:
-----------
javadoc fix.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java 2014-06-17 19:08:09 UTC (rev 8499)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java 2014-06-17 20:51:10 UTC (rev 8500)
@@ -915,10 +915,8 @@
* @param indexManager
* The {@link IIndexManager} that will be used to resolve the
* named indices for the relation.
- * @param namespace
- * The namespace for the relation.
- * @param timestamp
- * The timestamp for the view of the relation.
+ * @param nt
+ * The namespace and timestamp for the view of the relation.
* @param properties
* Configuration properties for the relation.
*
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-06-17 19:08:23
|
Revision: 8499
http://sourceforge.net/p/bigdata/code/8499
Author: tobycraig
Date: 2014-06-17 19:08:09 +0000 (Tue, 17 Jun 2014)
Log Message:
-----------
Fixed initial LBS state
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-17 19:02:35 UTC (rev 8498)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-17 19:08:09 UTC (rev 8499)
@@ -21,15 +21,19 @@
// debug to access closure variables
$('html, textarea, select').bind('keydown', 'ctrl+d', function() { debugger; });
-function useLBS() {
- if(this.checked) {
+function useLBS(state) {
+ // allows passing in of boolean, or firing on event
+ if(typeof(state) != 'boolean') {
+ state = this.checked;
+ }
+ if(state) {
RW_URL_PREFIX = '/bigdata/LBS/leader/';
RO_URL_PREFIX = '/bigdata/LBS/read/';
} else {
RW_URL_PREFIX = '/bigdata/';
RO_URL_PREFIX = '/bigdata/';
}
- $('.use-lbs').prop('checked', this.checked);
+ $('.use-lbs').prop('checked', state);
}
useLBS(true);
$('.use-lbs').change(useLBS);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-06-17 19:02:39
|
Revision: 8498
http://sourceforge.net/p/bigdata/code/8498
Author: tobycraig
Date: 2014-06-17 19:02:35 +0000 (Tue, 17 Jun 2014)
Log Message:
-----------
Added LBS toggle
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-06-17 18:51:24 UTC (rev 8497)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/index.html 2014-06-17 19:02:35 UTC (rev 8498)
@@ -65,6 +65,7 @@
<div class="advanced-features">
<input type="checkbox" id="update-analytic"> <label for="update-analytic">Analytic</label>
<input type="checkbox" id="update-monitor"> <label for="update-monitor">Monitor</label>
+ <input type="checkbox" id="use-lbs-update" class="use-lbs"> <label for="use-lbs-update">LBS</label>
</div>
<hr class="shadow">
<button id="update-update">Update</button>
@@ -98,6 +99,7 @@
<input type="checkbox" id="query-details"> <label for="query-explain">(Details)</label>
<input type="checkbox" name="analytic" value="true" id="query-analytic"> <label for="query-analytic">Analytic</label>
<input type="checkbox" name="RTO" value="true" id="query-rto"> <label for="query-rto">Runtime Query Optimizer</label>
+ <input type="checkbox" id="use-lbs-query" class="use-lbs"> <label for="use-lbs-query">LBS</label>
</div>
<hr class="shadow">
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-17 18:51:24 UTC (rev 8497)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-17 19:02:35 UTC (rev 8498)
@@ -1,13 +1,7 @@
$(function() {
// global variables
-var DEFAULT_NAMESPACE, NAMESPACE, NAMESPACES_READY, NAMESPACE_SHORTCUTS, FILE_CONTENTS, QUERY_RESULTS;
-// LBS URLs do not currently work with non-HA and HA1 setups. Set this to true to use LBS URLs
-if(false) {
- var RW_URL_PREFIX = '/bigdata/LBS/leader/', RO_URL_PREFIX = '/bigdata/LBS/read/';
-} else {
- var RW_URL_PREFIX = '/bigdata/', RO_URL_PREFIX = '/bigdata/';
-}
+var RW_URL_PREFIX, RO_URL_PREFIX, DEFAULT_NAMESPACE, NAMESPACE, NAMESPACES_READY, NAMESPACE_SHORTCUTS, FILE_CONTENTS, QUERY_RESULTS;
var CODEMIRROR_DEFAULTS, EDITORS = {}, ERROR_LINE_MARKERS = {}, ERROR_CHARACTER_MARKERS = {};
var PAGE_SIZE = 50, TOTAL_PAGES, CURRENT_PAGE;
var NAMESPACE_PARAMS = {
@@ -27,6 +21,19 @@
// debug to access closure variables
$('html, textarea, select').bind('keydown', 'ctrl+d', function() { debugger; });
+function useLBS() {
+ if(this.checked) {
+ RW_URL_PREFIX = '/bigdata/LBS/leader/';
+ RO_URL_PREFIX = '/bigdata/LBS/read/';
+ } else {
+ RW_URL_PREFIX = '/bigdata/';
+ RO_URL_PREFIX = '/bigdata/';
+ }
+ $('.use-lbs').prop('checked', this.checked);
+}
+useLBS(true);
+$('.use-lbs').change(useLBS);
+
/* Modal functions */
function showModal(id) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-06-17 18:51:34
|
Revision: 8497
http://sourceforge.net/p/bigdata/code/8497
Author: tobycraig
Date: 2014-06-17 18:51:24 +0000 (Tue, 17 Jun 2014)
Log Message:
-----------
Fixed typo
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-17 14:35:35 UTC (rev 8496)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-17 18:51:24 UTC (rev 8497)
@@ -285,7 +285,7 @@
'skos': 'http://www.w3.org/2004/02/skos/core#',
'xsd': 'http://www.w3.org/2001/XMLSchema#'
},
- 'Dublic Core': {
+ 'Dublin Core': {
'dc': 'http://purl.org/dc/elements/1.1/',
'dcterm': 'http://purl.org/dc/terms/',
'void': 'http://rdfs.org/ns/void#'
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2014-06-17 14:35:43
|
Revision: 8496
http://sourceforge.net/p/bigdata/code/8496
Author: martyncutcher
Date: 2014-06-17 14:35:35 +0000 (Tue, 17 Jun 2014)
Log Message:
-----------
Ensure main terminates cleanly - forcing System.exit() to prevent zombie threads keeping process alive.
Modify long-running-reader to be randomly selected from reader tasks.
Modified Paths:
--------------
branches/BIGDATA_ASYNC_WCS/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactions.java
Modified: branches/BIGDATA_ASYNC_WCS/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactions.java
===================================================================
--- branches/BIGDATA_ASYNC_WCS/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactions.java 2014-06-17 13:29:50 UTC (rev 8495)
+++ branches/BIGDATA_ASYNC_WCS/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactions.java 2014-06-17 14:35:35 UTC (rev 8496)
@@ -202,10 +202,14 @@
maxAborts, failex, subs, preds));
}
-
+
+ // Long Running Reader Index
+ final int lrrIndex = r.nextInt(nreaders);
+
for (int rdrs = 0; rdrs < nreaders; rdrs++) {
- final int nreads = rdrs == 0 ? Integer.MAX_VALUE : 60; // reasonably long running hopefully
+ // Is this reader long running?
+ final int nreads = rdrs == lrrIndex ? Integer.MAX_VALUE : 60;
lastReaderFuture = readers.submit(new Reader(r,
nreads, nwriters, sail, failex,
@@ -659,7 +663,8 @@
@Override
protected long writeNodeOrLeaf(final AbstractNode<?> node) {
- if (node.isLeaf() && r.nextInt(500) == 0) {
+ // if (node.isLeaf() && r.nextInt(500) == 0) {
+ if (node.isLeaf() && r.nextInt(5000) == 0) {
throw new MyBTreeException("Forcing abort: " + this);
@@ -779,6 +784,12 @@
}
sailShutdown.interrupt();
+
+ if (sail.get().isOpen())
+ sail.get().shutDown();
+
+ // Hard Exit
+ System.exit(0);
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-06-17 13:29:53
|
Revision: 8495
http://sourceforge.net/p/bigdata/code/8495
Author: thompsonbry
Date: 2014-06-17 13:29:50 +0000 (Tue, 17 Jun 2014)
Log Message:
-----------
HA service start changes:
- startHAServices: Now requires explicit configuration of LOCATORS and ZK_SERVERS. Now passes through the WRITE_CACHE_BUFFER_COUNT environment variable. This is used by the HAJournal.config.
- bigdataHA: Now specifies JVM_OPTS IFF they are not specified in the environment. This allows them to be overridden by the caller. If you want to have the defaults, then "unset" the JVM_OPTS variable in the environment before invoking this script.
See #965 (HA1 LBS)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices
branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices 2014-06-17 13:28:07 UTC (rev 8494)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/startHAServices 2014-06-17 13:29:50 UTC (rev 8495)
@@ -54,10 +54,14 @@
export GROUPS="$FEDNAME"
fi
if [ -z "${LOCATORS}" ]; then
- export LOCATORS="jini://bigdata15/,jini://bigdata16/,jini://bigdata17/"
+# export LOCATORS="jini://bigdata15/,jini://bigdata16/,jini://bigdata17/"
+ echo "Must specify: LOCATORS"
+ exit 1
fi
if [ -z "${ZK_SERVERS}" ]; then
- export ZK_SERVERS="bigdata15:2081,bigdata16:2081,bigdata17:2081";
+# export ZK_SERVERS="bigdata15:2081,bigdata16:2081,bigdata17:2081"
+ echo "Must specify: ZK_SERVERS"
+ exit 1
fi
export HA_OPTS="\
@@ -71,6 +75,7 @@
-DZK_SERVERS=${ZK_SERVERS}\
-DRMI_PORT=${RMI_PORT}\
-DHA_PORT=${HA_PORT}\
+ -DWRITE_CACHE_BUFFER_COUNT=${WRITE_CACHE_BUFFER_COUNT}\
"-Dcom.bigdata.hostname=${BIGDATA_HOSTNAME}"\
"-Djetty.port=${JETTY_PORT}"\
"-Djetty.threads.min=${JETTY_THREADS_MIN}"\
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA 2014-06-17 13:28:07 UTC (rev 8494)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/etc/default/bigdataHA 2014-06-17 13:29:50 UTC (rev 8495)
@@ -22,8 +22,13 @@
# size of the direct memory heap (used for the write cache buffers and
# some related things).
##
-export JVM_OPTS="-server -Xmx4G -XX:MaxDirectMemorySize=3000m"
-#export JVM_OPTS="-server -Xmx4G -XX:MaxDirectMemorySize=3000m -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=1046"
+if [ -z "$JVM_OPTS" ]; then
+ export JVM_OPTS="-server -Xmx4G -XX:MaxDirectMemorySize=3000m"
+# export JVM_OPTS="-server -Xmx4G -XX:MaxDirectMemorySize=3000m -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=1046"
+ echo "Set JVM_OPTS=${JVM_OPTS}"
+else
+ echo "Using JVM_OPTS=${JVM_OPTS}"
+fi
##
# The following variables configure the startHAServices script, which
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-06-17 13:28:15
|
Revision: 8494
http://sourceforge.net/p/bigdata/code/8494
Author: thompsonbry
Date: 2014-06-17 13:28:07 +0000 (Tue, 17 Jun 2014)
Log Message:
-----------
Fix for #965 (LBS does not work with HA1)
The root cause was the init() method on HALoadBalancerServlet. init() disabled the LBS unless the NSS was in an HA GT 1 deployment model. I have modified the code to always enable the LBS servlet. This allows it to correctly rewrite itself out of the request when in the HA1 or non-HA modes.
There are some code paths that need to be updated because either (a) they do not make an explicit choice about whether or not to use the LBS; (b) they do not parametrize the ContextPath of the web application; or (c) they are in the wrong package (client code should be in a separate patch from server code).
Changes are to:
- HALoadBalancerServlet: init() always succeeds. This fixes the core problem for this ticket.
- TestHA1JournalServer: the test is linked to this ticket.
- RemoteServiceCallImpl: Modified to use declared service configuration information to decide whether or not to use the LBS pattern for the remote service. The default is false, which works for all cases. The default may be overridden to be true if the end point is known to expose the bigdata LBS pattern.
- IServiceOptions: added the isBigdataLBS() method.
- ServiceOptionsBase: added default=false for isBigdataLBS() and setBigdataLBS() methods.
- BigdataSailFactory: Added "FIXME" - This does not support the HA load balancer pattern and does not parameterize the value of the ContextPath. Also, should this class be part of the "client" package?
- BigdataSailRemoteRepository: added constructor variant that accepts the boolean useLBS argument. The version of the constructor without that argument now defaults to useLBS:=true. This changes the default behavior of the client(!).
- RemoteRepository: deprecated the constructor version that does not accept the useLBS parameter. This version of the constructor still default to useLBS:=false. It tends to be used from some less common code paths.
- RemoteRepositoryManager: modified to specify useLBS:=true by default.
- TestNSSHealthCheck: code cleanup.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1JournalServer.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/INativeServiceOptions.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/IServiceOptions.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/RemoteServiceCallImpl.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceCallCreateParams.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceOptionsBase.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailFactory.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/remote/BigdataSailRemoteRepository.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepository.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepositoryManager.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/health/TestNSSHealthCheck.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1JournalServer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1JournalServer.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1JournalServer.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -123,17 +123,24 @@
}
+ /**
+ * A simple transaction test against an HA1 mode server using the LBS.
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/965" > Cannot run queries in
+ * LBS mode with HA1 setup </a>
+ */
public void testSimpleTransactionLBS() throws Exception {
-
- doStartA();
-
- serverA.awaitHAReady(2, TimeUnit.SECONDS);
-
- awaitCommitCounter(1, new HAGlue[] { serverA });
-
- simpleTransactionLBS();
-
- awaitCommitCounter(2, new HAGlue[] { serverA });
+
+ doStartA();
+
+ serverA.awaitHAReady(2, TimeUnit.SECONDS);
+
+ awaitCommitCounter(1, new HAGlue[] { serverA });
+
+ simpleTransactionLBS();
+
+ awaitCommitCounter(2, new HAGlue[] { serverA });
+
}
public void testMultiTransaction() throws Exception {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/INativeServiceOptions.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/INativeServiceOptions.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/INativeServiceOptions.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -32,7 +32,6 @@
* Additional options for native services.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public interface INativeServiceOptions extends IServiceOptions {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/IServiceOptions.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/IServiceOptions.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/IServiceOptions.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -37,7 +37,6 @@
* Options and metadata for service end points.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public interface IServiceOptions {
@@ -78,5 +77,13 @@
* query planner has locked in the join evaluation order.
*/
boolean isRunFirst();
+
+ /**
+ * Return <code>true</code> if the remote service is known to be a bigdata
+ * service that exposes the HA load balancer servlet (default
+ * <code>false</code>). The default may be overridden iff the end point is
+ * known to expose the bigdata LBS pattern.
+ */
+ boolean isBigdataLBS();
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/RemoteServiceCallImpl.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/RemoteServiceCallImpl.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/RemoteServiceCallImpl.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -29,7 +29,6 @@
import java.util.UUID;
-import org.apache.http.HttpResponse;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.openrdf.query.BindingSet;
@@ -49,8 +48,6 @@
* adjusting the {@link RemoteServiceOptions} for the service URI.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id: RemoteServiceCallImpl.java 6060 2012-03-02 16:07:38Z
- * thompsonbry $
*/
public class RemoteServiceCallImpl implements RemoteServiceCall {
@@ -149,7 +146,9 @@
// Setup a standard strategy for following redirects.
httpClient.setRedirectStrategy(new DefaultRedirectStrategy());
- final RemoteRepository repo = new RemoteRepository(uriStr,//
+ final RemoteRepository repo = new RemoteRepository(//
+ uriStr,//
+ params.getServiceOptions().isBigdataLBS(),// useLBS
httpClient,//
params.getTripleStore().getExecutorService()
);
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceCallCreateParams.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceCallCreateParams.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceCallCreateParams.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -37,7 +37,6 @@
* {@link ServiceCall} instance.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public interface ServiceCallCreateParams {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceOptionsBase.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceOptionsBase.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceOptionsBase.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -29,19 +29,27 @@
/**
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
abstract public class ServiceOptionsBase implements IServiceOptions {
private boolean isRunFirst = false;
-
+ private boolean useLBS = false;
+
@Override
public boolean isRunFirst() {
return isRunFirst;
}
-
- public void setRunFirst(final boolean newValue) {
+
+ public void setRunFirst(final boolean newValue) {
this.isRunFirst = newValue;
}
+ @Override
+ public boolean isBigdataLBS() {
+ return useLBS;
+ }
+
+ public void setBigdataLBS(final boolean newValue) {
+ this.useLBS = newValue;
+ }
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailFactory.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailFactory.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailFactory.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -96,6 +96,10 @@
/**
* Connect to a remote bigdata instance.
+ *
+ * FIXME This does not support the HA load balancer pattern and does not
+ * parameterize the value of the ContextPath. Also, should this class be
+ * part of the "client" package?
*/
public static BigdataSailRemoteRepository connect(final String serviceEndpoint) {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/remote/BigdataSailRemoteRepository.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/remote/BigdataSailRemoteRepository.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/remote/BigdataSailRemoteRepository.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -1,5 +1,3 @@
-package com.bigdata.rdf.sail.remote;
-
/**
Copyright (C) SYSTAP, LLC 2006-2012. All rights reserved.
@@ -24,6 +22,8 @@
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
+package com.bigdata.rdf.sail.remote;
+
import java.io.File;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -66,9 +66,30 @@
/**
* Ctor that simply specifies an endpoint and lets this class manage the
* ClientConnectionManager for the HTTP client and the manage the
- * ExecutorService. More convenient.
+ * ExecutorService. More convenient, but does not account for whether or not
+ * to use the LBS.
+ *
+ * @param sparqlEndpointURL
+ * The SPARQL end point URL
*/
- public BigdataSailRemoteRepository(final String sparqlEndpointURL) {
+ public BigdataSailRemoteRepository(final String sparqlEndpointURL) {
+
+ this(sparqlEndpointURL, true/* useLBS */);
+
+ }
+
+ /**
+ * Ctor that simply specifies an endpoint and lets this class manage the
+ * ClientConnectionManager for the HTTP client and the manage the
+ * ExecutorService.
+ *
+ * @param sparqlEndpointURL
+ * The SPARQL end point URL
+ * @param useLBS
+ * <code>true</code> iff the LBS pattern should be used.
+ */
+ public BigdataSailRemoteRepository(final String sparqlEndpointURL,
+ final boolean useLBS) {
this.executor = Executors.newCachedThreadPool();
@@ -84,9 +105,9 @@
*/
httpClient.setRedirectStrategy(new DefaultRedirectStrategy());
- this.nss = new RemoteRepository(
- sparqlEndpointURL, httpClient, executor);
-
+ this.nss = new RemoteRepository(sparqlEndpointURL, useLBS, httpClient,
+ executor);
+
}
/**
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HALoadBalancerServlet.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -386,8 +386,15 @@
* {@inheritDoc}
* <p>
* Extended to setup the as-configured {@link IHALoadBalancerPolicy}.
+ * <p>
+ * Note: If the deployment is does not support HA replication (e.g., either
+ * not HA or HA with replicationFactor:=1), then we still want to be able to
+ * forward to the local service.
*
* @throws ServletException
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/965" > Cannot run queries in
+ * LBS mode with HA1 setup </a>
*/
@Override
public void init() throws ServletException {
@@ -405,42 +412,35 @@
final IIndexManager indexManager = BigdataServlet
.getIndexManager(servletContext);
- if (!(indexManager instanceof HAJournal)){
- // This is not an error, but the LBS is only for HA.
- log.info("LBS Disabled - not HA");
- return;
- }
- if (indexManager instanceof AbstractJournal
+ if (indexManager instanceof HAJournal
&& ((AbstractJournal) indexManager).getQuorum() != null
&& ((AbstractJournal) indexManager).getQuorum()
- .replicationFactor() == 1) {
- // This is not an error, but the LBS is only for HA.
- log.info("LBS Disabled - not HA");
- return;
- }
+ .replicationFactor() > 1) {
- {
- // Get the as-configured policy.
- final IHALoadBalancerPolicy policy = newInstance(//
- servletConfig, //
- HALoadBalancerServlet.class,// owningClass
- IHALoadBalancerPolicy.class,//
- InitParams.POLICY, InitParams.DEFAULT_POLICY);
+ {
+ // Get the as-configured policy.
+ final IHALoadBalancerPolicy policy = newInstance(//
+ servletConfig, //
+ HALoadBalancerServlet.class,// owningClass
+ IHALoadBalancerPolicy.class,//
+ InitParams.POLICY, InitParams.DEFAULT_POLICY);
- // Set the as-configured policy.
- setLBSPolicy(policy);
+ // Set the as-configured policy.
+ setLBSPolicy(policy);
- }
- {
+ }
+ {
- final IHARequestURIRewriter rewriter = newInstance(//
- servletConfig,//
- HALoadBalancerServlet.class, // owningClass
- IHARequestURIRewriter.class,//
- InitParams.REWRITER, InitParams.DEFAULT_REWRITER);
+ final IHARequestURIRewriter rewriter = newInstance(//
+ servletConfig,//
+ HALoadBalancerServlet.class, // owningClass
+ IHARequestURIRewriter.class,//
+ InitParams.REWRITER, InitParams.DEFAULT_REWRITER);
- setRewriter(rewriter);
+ setRewriter(rewriter);
+ }
+
}
servletContext.setAttribute(BigdataServlet.ATTRIBUTE_LBS_PREFIX,
@@ -850,6 +850,10 @@
* the request to the servlet at the resulting requestURI. This forwarding
* effectively disables the LBS but still allows requests which target the
* LBS to succeed against the webapp on the same host.
+ * <p>
+ * Note: If the deployment is does not support HA replication (e.g., either
+ * not HA or HA with replicationFactor:=1), then we still want to be able to
+ * forward to the local service.
*
* @param request
* The request.
@@ -858,6 +862,9 @@
*
* @throws IOException
* @throws ServletException
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/965" > Cannot run queries in
+ * LBS mode with HA1 setup </a>
*/
public void forwardToLocalService(//
final boolean isLeaderRequest,//
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepository.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepository.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepository.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -307,6 +307,20 @@
}
+ /**
+ *
+ * @param sparqlEndpointURL
+ * @param httpClient
+ * @param executor
+ *
+ * @deprecated This version does not force the caller to decide whether or
+ * not the LBS pattern will be used. In general, it should be
+ * used if the end point is bigdata. This class is generally,
+ * but not always, used with a bigdata end point. The main
+ * exception is SPARQL Basic Federated Query. For that use case
+ * we can not assume that the end point is bigdata and thus we
+ * can not use the LBS prefix.
+ */
public RemoteRepository(final String sparqlEndpointURL,
final HttpClient httpClient, final Executor executor) {
@@ -865,7 +879,7 @@
}
if (add.uri != null) {
- // set the resource to load.
+ // set the resource to load : FIXME REST API allows multiple URIs, but RemoteRepository does not.
opts.addRequestParam("uri", add.uri);
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepositoryManager.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepositoryManager.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/client/RemoteRepositoryManager.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -89,7 +89,7 @@
public RemoteRepositoryManager(final String serviceURL,
final HttpClient httpClient, final Executor executor) {
- this(serviceURL, false/* useLBS */, httpClient, executor);
+ this(serviceURL, true/* useLBS */, httpClient, executor);
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/health/TestNSSHealthCheck.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/health/TestNSSHealthCheck.java 2014-06-17 12:17:24 UTC (rev 8493)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/health/TestNSSHealthCheck.java 2014-06-17 13:28:07 UTC (rev 8494)
@@ -375,9 +375,9 @@
*
* @return The connection.
*
- * @see <a href="https://sourceforge.net/apps/trac/bigdata/ticket/619">
- * RemoteRepository class should use application/x-www-form-urlencoded
- * for large POST requests </a>
+ * @see <a href="http://trac.bigdata.com/ticket/619"> RemoteRepository class
+ * should use application/x-www-form-urlencoded for large POST requests
+ * </a>
*/
private HttpResponse doConnect(final ConnectOptions opts) throws Exception {
@@ -452,57 +452,17 @@
}
-// // conn = doConnect(urlString.toString(), opts.method);
-// final URL url = new URL(urlString.toString());
-// conn = (HttpURLConnection) url.openConnection();
-// conn.setRequestMethod(opts.method);
-// conn.setDoOutput(true);
-// conn.setDoInput(true);
-// conn.setUseCaches(false);
-// conn.setReadTimeout(opts.timeout);
-// conn.setRequestProperty("Accept", opts.acceptHeader);
-// if (log.isDebugEnabled())
-// log.debug("Accept: " + opts.acceptHeader);
-
if (opts.entity != null) {
-// if (opts.data == null)
-// throw new AssertionError();
-
-// final String contentLength = Integer.toString(opts.data.length);
-
-// conn.setRequestProperty("Content-Type", opts.contentType);
-// conn.setRequestProperty("Content-Length", contentLength);
-
-// if (log.isDebugEnabled()) {
-// log.debug("Content-Type: " + opts.contentType);
-// log.debug("Content-Length: " + contentLength);
-// }
-
-// final ByteArrayEntity entity = new ByteArrayEntity(opts.data);
-// entity.setContentType(opts.contentType);
-
- ((HttpEntityEnclosingRequestBase) request).setEntity(opts.entity);
+ ((HttpEntityEnclosingRequestBase) request)
+ .setEntity(opts.entity);
-// final OutputStream os = conn.getOutputStream();
-// try {
-// os.write(opts.data);
-// os.flush();
-// } finally {
-// os.close();
-// }
-
}
final HttpResponse response = m_httpClient.execute(request);
return response;
-// // connect.
-// conn.connect();
-//
-// return conn;
-
} catch (Throwable t) {
/*
* If something goes wrong, then close the http connection.
@@ -513,10 +473,6 @@
if (request != null)
request.abort();
-// // clean up the connection resources
-// if (conn != null)
-// conn.disconnect();
-
} catch (Throwable t2) {
// ignored.
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-06-17 12:17:28
|
Revision: 8493
http://sourceforge.net/p/bigdata/code/8493
Author: thompsonbry
Date: 2014-06-17 12:17:24 +0000 (Tue, 17 Jun 2014)
Log Message:
-----------
bug fix: snapshot => snapshotDir in HAJournal.config for ant stage and related targets.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config
Modified: branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config 2014-06-17 12:16:47 UTC (rev 8492)
+++ branches/BIGDATA_RELEASE_1_3_0/src/resources/HAJournal/HAJournal.config 2014-06-17 12:17:24 UTC (rev 8493)
@@ -93,7 +93,7 @@
//private static haLogDir = new File(serviceDir,"HALog");
// Snapshot directory.
- private static snapshot = new File(ConfigMath.getProperty("SNAPSHOT_DIR",""+serviceDir+File.separator+"snapshot"));
+ private static snapshotDir = new File(ConfigMath.getProperty("SNAPSHOT_DIR",""+serviceDir+File.separator+"snapshot"));
//private static snapshotDir = new File(serviceDir,"snapshot");
/* Snapshot policy. Choose one.
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-06-17 12:16:57
|
Revision: 8492
http://sourceforge.net/p/bigdata/code/8492
Author: thompsonbry
Date: 2014-06-17 12:16:47 +0000 (Tue, 17 Jun 2014)
Log Message:
-----------
Changed the max form size for jetty to 10M rather than the somewhat strange default it was given.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/jetty.xml
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/jetty.xml
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/jetty.xml 2014-06-16 22:10:27 UTC (rev 8491)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/jetty.xml 2014-06-17 12:16:47 UTC (rev 8492)
@@ -148,7 +148,7 @@
<Set name="parentLoaderPriority">true</Set>
<Set name="extractWAR">false</Set>
<Set name="overrideDescriptor"><SystemProperty name="jetty.overrideWebXml" default="bigdata-war/src/WEB-INF/override-web.xml"/></Set>
- <Set name="maxFormContentSize">105000000</Set>
+ <Set name="maxFormContentSize">10485760</Set>
</New>
</Arg>
</Call>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-06-16 22:10:34
|
Revision: 8491
http://sourceforge.net/p/bigdata/code/8491
Author: tobycraig
Date: 2014-06-16 22:10:27 +0000 (Mon, 16 Jun 2014)
Log Message:
-----------
Added test for HA1 with LBS
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3JournalServerTestCase.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1JournalServer.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3JournalServerTestCase.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3JournalServerTestCase.java 2014-06-16 21:12:25 UTC (rev 8490)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3JournalServerTestCase.java 2014-06-16 22:10:27 UTC (rev 8491)
@@ -2807,6 +2807,25 @@
}
/**
+ * Commits update transaction with LBS after awaiting quorum.
+ */
+ protected void simpleTransactionLBS() throws IOException, Exception {
+
+ // Await quorum meet.
+ final long token = quorum.awaitQuorum(awaitQuorumTimeout,
+ TimeUnit.MILLISECONDS);
+
+ // Figure out which service is the leader.
+ final HAGlue leader = quorum.getClient().getLeader(token);
+
+ // Wait until that service is ready to act as the leader.
+ assertEquals(HAStatusEnum.Leader, awaitNSSAndHAReady(leader));
+
+ simpleTransaction_noQuorumCheckLBS(leader);
+
+ }
+
+ /**
* Immediately issues a simple transaction against the service.
*
* @param leader
@@ -2823,6 +2842,22 @@
}
/**
+ * Immediately issues a simple transaction against the service with LBS.
+ *
+ * @param leader
+ * The service (must be the leader to succeed).
+ *
+ * @throws IOException
+ * @throws Exception
+ */
+ protected void simpleTransaction_noQuorumCheckLBS(final HAGlue leader)
+ throws IOException, Exception {
+
+ simpleTransaction_noQuorumCheck(leader, true/* useLoadBalancer */);
+
+ }
+
+ /**
* Immediately issues a simple transaction against the service.
*
* @param haGlue
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1JournalServer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1JournalServer.java 2014-06-16 21:12:25 UTC (rev 8490)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1JournalServer.java 2014-06-16 22:10:27 UTC (rev 8491)
@@ -123,6 +123,19 @@
}
+ public void testSimpleTransactionLBS() throws Exception {
+
+ doStartA();
+
+ serverA.awaitHAReady(2, TimeUnit.SECONDS);
+
+ awaitCommitCounter(1, new HAGlue[] { serverA });
+
+ simpleTransactionLBS();
+
+ awaitCommitCounter(2, new HAGlue[] { serverA });
+ }
+
public void testMultiTransaction() throws Exception {
doStartA();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-06-16 21:12:33
|
Revision: 8490
http://sourceforge.net/p/bigdata/code/8490
Author: tobycraig
Date: 2014-06-16 21:12:25 +0000 (Mon, 16 Jun 2014)
Log Message:
-----------
Added max form size jetty setting to accommodate 10mb files
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/jetty.xml
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/jetty.xml
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/jetty.xml 2014-06-16 21:02:08 UTC (rev 8489)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/jetty.xml 2014-06-16 21:12:25 UTC (rev 8490)
@@ -148,6 +148,7 @@
<Set name="parentLoaderPriority">true</Set>
<Set name="extractWAR">false</Set>
<Set name="overrideDescriptor"><SystemProperty name="jetty.overrideWebXml" default="bigdata-war/src/WEB-INF/override-web.xml"/></Set>
+ <Set name="maxFormContentSize">105000000</Set>
</New>
</Arg>
</Call>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-06-16 21:02:16
|
Revision: 8489
http://sourceforge.net/p/bigdata/code/8489
Author: tobycraig
Date: 2014-06-16 21:02:08 +0000 (Mon, 16 Jun 2014)
Log Message:
-----------
Fixed monitor update in Firefox
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-16 16:54:52 UTC (rev 8488)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-16 21:02:08 UTC (rev 8489)
@@ -533,14 +533,16 @@
// see if monitor mode is on
if($('#update-monitor').is(':checked')) {
// create form and submit it, sending output to the iframe
- var form = $('<form method="POST" target="update-response-container">')
+ var form = $('<form id="update-monitor-form" method="POST" target="update-response-container">')
.attr('action', url)
.append($('<input name="update">').val(settings.data))
.append('<input name="monitor" value="true">');
if($('#update-analytic').is(':checked')) {
form.append('<input name="analytic" value="true">')
}
+ form.appendTo($('body'));
form.submit();
+ $('#update-monitor-form').remove();
$('#update-response iframe, #update-clear-container').show();
$('#update-response pre').hide();
return;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tob...@us...> - 2014-06-16 16:55:02
|
Revision: 8488
http://sourceforge.net/p/bigdata/code/8488
Author: tobycraig
Date: 2014-06-16 16:54:52 +0000 (Mon, 16 Jun 2014)
Log Message:
-----------
Changed wording on update panel
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-16 16:25:19 UTC (rev 8487)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-war/src/html/js/workbench.js 2014-06-16 16:54:52 UTC (rev 8488)
@@ -568,7 +568,7 @@
break;
}
- $('#update-response pre').show().html('Data loading...');
+ $('#update-response pre').show().html('Running update...');
$.ajax(url, settings);
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-06-16 16:25:26
|
Revision: 8487
http://sourceforge.net/p/bigdata/code/8487
Author: thompsonbry
Date: 2014-06-16 16:25:19 +0000 (Mon, 16 Jun 2014)
Log Message:
-----------
Added support for IsolatedActionJournal.indexNameScan(). This is invoked by DROP ALL from the SPARQL layer if we are using group commit.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractTask.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractTask.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractTask.java 2014-06-16 16:23:25 UTC (rev 8486)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractTask.java 2014-06-16 16:25:19 UTC (rev 8487)
@@ -83,6 +83,9 @@
import com.bigdata.util.InnerCause;
import com.bigdata.util.concurrent.TaskCounters;
+import cutthecrap.utils.striterators.Resolver;
+import cutthecrap.utils.striterators.Striterator;
+
/**
* Abstract base class for tasks that may be submitted to the
* {@link ConcurrencyManager}. Tasks may be isolated (by a transaction),
@@ -292,7 +295,7 @@
* @param checkpointAddr
* @param commitTime
*/
- Entry(String name, long checkpointAddr, long commitTime) {
+ Entry(final String name, final long checkpointAddr, final long commitTime) {
super(name, checkpointAddr, commitTime);
@@ -3007,9 +3010,26 @@
return delegate.getHttpdPort();
}
+ /**
+ * {@inheritDoc}
+ * <p>
+ * Overridden to visit the name of all indices that were isolated and to
+ * ignore the timestamp.
+ */
@Override
- public Iterator<String> indexNameScan(String prefix, long timestamp) {
- throw new UnsupportedOperationException();
+ public Iterator<String> indexNameScan(final String prefix,
+ final long timestampIsIgnored) {
+
+ return new Striterator(n2a.values().iterator())
+ .addFilter(new Resolver() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected Object resolve(final Object obj) {
+ return ((Entry)obj).name;
+ }
+ });
+
}
@Override
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-06-16 16:23:32
|
Revision: 8486
http://sourceforge.net/p/bigdata/code/8486
Author: thompsonbry
Date: 2014-06-16 16:23:25 +0000 (Mon, 16 Jun 2014)
Log Message:
-----------
Correctness: should set success:=true immediately after the commit.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java 2014-06-16 16:22:55 UTC (rev 8485)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/InsertServlet.java 2014-06-16 16:23:25 UTC (rev 8486)
@@ -315,12 +315,12 @@
// Commit the mutation.
conn.commit();
+ success = true;
+
final long elapsed = System.currentTimeMillis() - begin;
reportModifiedCount(nmodified.get(), elapsed);
- success = true;
-
return (Void) null;
} finally {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-06-16 16:23:03
|
Revision: 8485
http://sourceforge.net/p/bigdata/code/8485
Author: thompsonbry
Date: 2014-06-16 16:22:55 +0000 (Mon, 16 Jun 2014)
Log Message:
-----------
Typo.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/ASTContainer.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/ASTContainer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/ASTContainer.java 2014-06-16 16:22:43 UTC (rev 8484)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/ASTContainer.java 2014-06-16 16:22:55 UTC (rev 8485)
@@ -122,7 +122,7 @@
}
/**
- * Return the original SPARQL QUERY -or- UPATE from which this AST model was
+ * Return the original SPARQL QUERY -or- UPDATE from which this AST model was
* generated.
*
* @return The original Query or Update -or- <code>null</code> if the AST
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
4 messages has been excluded from this view by a project administrator.
