Menu
▾
▴
bigdata-commit — Commit traffic.
This list is closed, nobody may subscribe to it.
| 2010 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
(139) |
Aug
(94) |
Sep
(232) |
Oct
(143) |
Nov
(138) |
Dec
(55) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2011 |
Jan
(127) |
Feb
(90) |
Mar
(101) |
Apr
(74) |
May
(148) |
Jun
(241) |
Jul
(169) |
Aug
(121) |
Sep
(157) |
Oct
(199) |
Nov
(281) |
Dec
(75) |
| 2012 |
Jan
(107) |
Feb
(122) |
Mar
(184) |
Apr
(73) |
May
(14) |
Jun
(49) |
Jul
(26) |
Aug
(103) |
Sep
(133) |
Oct
(61) |
Nov
(51) |
Dec
(55) |
| 2013 |
Jan
(59) |
Feb
(72) |
Mar
(99) |
Apr
(62) |
May
(92) |
Jun
(19) |
Jul
(31) |
Aug
(138) |
Sep
(47) |
Oct
(83) |
Nov
(95) |
Dec
(111) |
| 2014 |
Jan
(125) |
Feb
(60) |
Mar
(119) |
Apr
(136) |
May
(270) |
Jun
(83) |
Jul
(88) |
Aug
(30) |
Sep
(47) |
Oct
(27) |
Nov
(23) |
Dec
|
| 2015 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
(3) |
Oct
|
Nov
|
Dec
|
| 2016 |
Jan
|
Feb
|
Mar
(4) |
Apr
(1) |
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
|
From: <tho...@us...> - 2010-07-26 19:32:06
|
Revision: 3299
http://bigdata.svn.sourceforge.net/bigdata/?rev=3299&view=rev
Author: thompsonbry
Date: 2010-07-26 19:31:59 +0000 (Mon, 26 Jul 2010)
Log Message:
-----------
Added explicit version control to a number of Externalizable implementations.
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedure.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/mdi/IndexPartitionCause.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/SameVariableConstraint.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/DefaultRuleTaskFactory.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/service/proxy/RemoteAsynchronousIteratorImpl.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/service/proxy/RemoteChunk.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/AbstractAtomicRowReadOrWrite.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/axioms/BaseAxioms.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/RdfTypeRdfsResourceFilter.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdTupleSerializer.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/InGraphBinarySearchFilter.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/vocab/BaseVocabulary.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOValueCoders.java
Removed Paths:
-------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/FastRDFValueCoder.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedure.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedure.java 2010-07-26 19:14:39 UTC (rev 3298)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedure.java 2010-07-26 19:31:59 UTC (rev 3299)
@@ -677,6 +677,16 @@
public void readExternal(final ObjectInput in) throws IOException,
ClassNotFoundException {
+ final byte version = in.readByte();
+
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new UnsupportedOperationException("Unknown version: "
+ + version);
+ }
+
final InputBitStream ibs = new InputBitStream((InputStream) in,
0/* unbuffered */, false/* reflectionTest */);
@@ -699,6 +709,8 @@
public void writeExternal(final ObjectOutput out) throws IOException {
+ out.writeByte(VERSION);
+
final OutputBitStream obs = new OutputBitStream((OutputStream) out,
0/* unbuffered! */, false/*reflectionTest*/);
@@ -716,6 +728,16 @@
}
+ /**
+ * The initial version.
+ */
+ private static final transient byte VERSION0 = 0;
+
+ /**
+ * The current version.
+ */
+ private static final transient byte VERSION = VERSION0;
+
}
/**
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/mdi/IndexPartitionCause.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/mdi/IndexPartitionCause.java 2010-07-26 19:14:39 UTC (rev 3298)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/mdi/IndexPartitionCause.java 2010-07-26 19:31:59 UTC (rev 3299)
@@ -189,9 +189,24 @@
}
+ /**
+ * The initial version.
+ */
+ private static final transient byte VERSION0 = 0;
+
+ /**
+ * The current version.
+ */
+ private static final transient byte VERSION = VERSION0;
+
public void readExternal(ObjectInput in) throws IOException,
ClassNotFoundException {
+ final byte version = in.readByte();
+
+ if (version != VERSION0)
+ throw new UnsupportedOperationException("version=" + version);
+
cause = CauseEnum.valueOf(in.readByte());
synchronousOverflowCounter = in.readLong();
@@ -202,6 +217,7 @@
public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeByte(VERSION);
out.writeByte(cause.code);
out.writeLong(synchronousOverflowCounter);
out.writeLong(lastCommitTime);
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/SameVariableConstraint.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/SameVariableConstraint.java 2010-07-26 19:14:39 UTC (rev 3298)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/SameVariableConstraint.java 2010-07-26 19:31:59 UTC (rev 3299)
@@ -183,10 +183,30 @@
}
+ /**
+ * The initial version.
+ */
+ private static final transient byte VERSION0 = 0;
+
+ /**
+ * The current version.
+ */
+ private static final transient byte VERSION = VERSION0;
+
@SuppressWarnings("unchecked")
public void readExternal(ObjectInput in) throws IOException,
ClassNotFoundException {
+ final short version = in.readByte();
+
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new UnsupportedOperationException("Unknown version: "
+ + version);
+ }
+
p = (IPredicate<E>) in.readObject();
final int len = (int) LongPacker.unpackLong(in);
@@ -203,6 +223,8 @@
public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeByte(VERSION);
+
out.writeObject(p);
LongPacker.packLong(out, indices.length);
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/DefaultRuleTaskFactory.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/DefaultRuleTaskFactory.java 2010-07-26 19:14:39 UTC (rev 3298)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/DefaultRuleTaskFactory.java 2010-07-26 19:31:59 UTC (rev 3299)
@@ -134,15 +134,37 @@
}
+ /**
+ * The initial version.
+ */
+ private static final transient byte VERSION0 = 0;
+
+ /**
+ * The current version.
+ */
+ private static final transient byte VERSION = VERSION0;
+
public void readExternal(ObjectInput in) throws IOException,
ClassNotFoundException {
+ final byte version = in.readByte();
+
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new UnsupportedOperationException("Unknown version: "
+ + version);
+ }
+
subquery = in.readBoolean();
}
public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeByte(VERSION);
+
out.writeBoolean(subquery);
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/service/proxy/RemoteAsynchronousIteratorImpl.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/service/proxy/RemoteAsynchronousIteratorImpl.java 2010-07-26 19:14:39 UTC (rev 3298)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/service/proxy/RemoteAsynchronousIteratorImpl.java 2010-07-26 19:31:59 UTC (rev 3299)
@@ -195,8 +195,28 @@
}
+ /**
+ * The initial version.
+ */
+ private static final transient byte VERSION0 = 0;
+
+ /**
+ * The current version.
+ */
+ private static final transient byte VERSION = VERSION0;
+
public void readExternal(ObjectInput in) throws IOException,
ClassNotFoundException {
+
+ final byte version = in.readByte();
+
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new UnsupportedOperationException("Unknown version: "
+ + version);
+ }
ser = (IStreamSerializer<E>) in.readObject();
@@ -206,6 +226,8 @@
public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeByte(VERSION);
+
out.writeObject(ser);
ser.serialize(out, e);
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/service/proxy/RemoteChunk.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/service/proxy/RemoteChunk.java 2010-07-26 19:14:39 UTC (rev 3298)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/service/proxy/RemoteChunk.java 2010-07-26 19:31:59 UTC (rev 3299)
@@ -89,8 +89,28 @@
}
+ /**
+ * The initial version.
+ */
+ private static final transient byte VERSION0 = 0;
+
+ /**
+ * The current version.
+ */
+ private static final transient byte VERSION = VERSION0;
+
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ final byte version = in.readByte();
+
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new UnsupportedOperationException("Unknown version: "
+ + version);
+ }
+
// true iff exhausted.
exhausted = in.readBoolean();
@@ -114,6 +134,8 @@
public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeByte(VERSION);
+
out.writeBoolean(exhausted);
// true iff there are any elements in this chunk.
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/AbstractAtomicRowReadOrWrite.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/AbstractAtomicRowReadOrWrite.java 2010-07-26 19:14:39 UTC (rev 3298)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/AbstractAtomicRowReadOrWrite.java 2010-07-26 19:31:59 UTC (rev 3299)
@@ -426,9 +426,29 @@
}
+ /**
+ * The initial version.
+ */
+ private static final transient byte VERSION0 = 0;
+
+ /**
+ * The current version.
+ */
+ private static final transient byte VERSION = VERSION0;
+
public void readExternal(ObjectInput in) throws IOException,
ClassNotFoundException {
+ final byte version = in.readByte();
+
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new UnsupportedOperationException("Unknown version: "
+ + version);
+ }
+
schema = (Schema) in.readObject();
primaryKey = in.readObject();
@@ -442,7 +462,9 @@
}
public void writeExternal(ObjectOutput out) throws IOException {
-
+
+ out.writeByte(VERSION);
+
out.writeObject(schema);
out.writeObject(primaryKey);
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/axioms/BaseAxioms.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/axioms/BaseAxioms.java 2010-07-26 19:14:39 UTC (rev 3298)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/axioms/BaseAxioms.java 2010-07-26 19:31:59 UTC (rev 3299)
@@ -294,8 +294,29 @@
}
- public void readExternal(final ObjectInput in) throws IOException, ClassNotFoundException {
-
+ /**
+ * The initial version.
+ */
+ private static final transient byte VERSION0 = 0;
+
+ /**
+ * The current version.
+ */
+ private static final transient byte VERSION = VERSION0;
+
+ public void readExternal(final ObjectInput in) throws IOException,
+ ClassNotFoundException {
+
+ final byte version = in.readByte();
+
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new UnsupportedOperationException("Unknown version: "
+ + version);
+ }
+
final long naxioms = LongPacker.unpackLong(in);
if (naxioms < 0 || naxioms > Integer.MAX_VALUE)
@@ -332,6 +353,8 @@
if (btree == null)
throw new IllegalStateException();
+ out.writeByte(VERSION);
+
final long naxioms = btree.rangeCount();
LongPacker.packLong(out, naxioms);
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/RdfTypeRdfsResourceFilter.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/RdfTypeRdfsResourceFilter.java 2010-07-26 19:14:39 UTC (rev 3298)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/RdfTypeRdfsResourceFilter.java 2010-07-26 19:31:59 UTC (rev 3299)
@@ -70,8 +70,28 @@
}
+ /**
+ * The initial version.
+ */
+ private static final transient short VERSION0 = 0;
+
+ /**
+ * The current version.
+ */
+ private static final transient short VERSION = VERSION0;
+
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+
+ final short version = in.readShort();
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new UnsupportedOperationException("Unknown version: "
+ + version);
+ }
+
// rdfType = LongPacker.unpackLong(in);
//
// rdfsResource = LongPacker.unpackLong(in);
@@ -84,6 +104,8 @@
public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeShort(VERSION);
+
// LongPacker.packLong(out,rdfType);
//
// LongPacker.packLong(out,rdfsResource);
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdTupleSerializer.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdTupleSerializer.java 2010-07-26 19:14:39 UTC (rev 3298)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdTupleSerializer.java 2010-07-26 19:31:59 UTC (rev 3299)
@@ -48,6 +48,8 @@
package com.bigdata.rdf.lexicon;
import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
import java.util.Properties;
import org.openrdf.model.Value;
import com.bigdata.btree.DefaultTupleSerializer;
@@ -55,6 +57,8 @@
import com.bigdata.btree.keys.DefaultKeyBuilderFactory;
import com.bigdata.btree.keys.IKeyBuilderFactory;
import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.btree.keys.ThreadLocalKeyBuilderFactory;
+import com.bigdata.btree.raba.codec.IRabaCoder;
import com.bigdata.io.DataOutputBuffer;
import com.bigdata.rawstore.Bytes;
import com.bigdata.rdf.internal.IV;
@@ -205,4 +209,33 @@
}
+ /**
+ * The initial version (no additional persistent state).
+ */
+ private final static transient int VERSION0 = 0;
+
+ /**
+ * The current version.
+ */
+ private final static transient int VERSION = VERSION0;
+
+ public void readExternal(final ObjectInput in) throws IOException,
+ ClassNotFoundException {
+ super.readExternal(in);
+ final short version = in.readShort();
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new UnsupportedOperationException("Unknown version: "
+ + version);
+ }
+
+ }
+
+ public void writeExternal(final ObjectOutput out) throws IOException {
+ super.writeExternal(out);
+ out.writeShort(VERSION);
+ }
+
}
Deleted: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/FastRDFValueCoder.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/FastRDFValueCoder.java 2010-07-26 19:14:39 UTC (rev 3298)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/FastRDFValueCoder.java 2010-07-26 19:31:59 UTC (rev 3299)
@@ -1,474 +0,0 @@
-package com.bigdata.rdf.spo;
-
-import it.unimi.dsi.io.InputBitStream;
-import it.unimi.dsi.io.OutputBitStream;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.io.OutputStream;
-
-import org.apache.log4j.Logger;
-
-import com.bigdata.btree.BytesUtil;
-import com.bigdata.btree.raba.IRaba;
-import com.bigdata.btree.raba.codec.AbstractCodedRaba;
-import com.bigdata.btree.raba.codec.IRabaCoder;
-import com.bigdata.btree.raba.codec.ICodedRaba;
-import com.bigdata.io.AbstractFixedByteArrayBuffer;
-import com.bigdata.io.DataOutputBuffer;
-import com.bigdata.rawstore.Bytes;
-import com.bigdata.rdf.model.StatementEnum;
-import com.bigdata.rdf.store.AbstractTripleStore;
-
-/**
- * Coder for statement index with inference enabled but without SIDS. We encode
- * the value in 3 bits per statement. The 1st bit is the override flag. The
- * remaining two bits are the statement type {inferred, explicit, or axiom}. The
- * bit sequence <code>111</code> is used as a place holder for a
- * <code>null</code> value and de-serializes to a [null].
- * <p>
- * Note: the 'override' flag is NOT stored in the statement indices, but it is
- * passed by the procedure that writes on the statement indices so that we can
- * decide whether or not to override the type when the statement is pre-existing
- * in the index.
- * <p>
- * Note: this procedure can not be used if
- * {@link AbstractTripleStore.Options#STATEMENT_IDENTIFIERS} are enabled.
- *
- * @see StatementEnum
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- *
- * @deprecated by {@link FastRDFValueCoder2}
- */
-public class FastRDFValueCoder implements Externalizable, IRabaCoder {
-
- protected static final Logger log = Logger
- .getLogger(FastRDFValueCoder.class);
-
- /**
- *
- */
- private static final long serialVersionUID = 1933430721504168533L;
-
- /**
- * The only version defined so far.
- */
- private static transient final byte VERSION0 = 0x00;
-
- /**
- * The bit offset of the start of the coded values. Each value is 3 bits.
- */
- private static transient final long O_values = (1/* version */+ Bytes.SIZEOF_INT/* size */) << 3;
-
- /**
- * No.
- */
- final public boolean isKeyCoder() {
-
- return false;
-
- }
-
- /**
- * Yes.
- */
- final public boolean isValueCoder() {
-
- return true;
-
- }
-
- /**
- * Sole constructor (handles de-serialization also).
- */
- public FastRDFValueCoder() {
-
- }
-
- public void writeExternal(ObjectOutput out) throws IOException {
-
- // NOP
-
- }
-
- public void readExternal(ObjectInput in) throws IOException,
- ClassNotFoundException {
-
- // NOP
-
- }
-
- public AbstractFixedByteArrayBuffer encode(final IRaba raba,
- final DataOutputBuffer buf) {
-
- /*
- * Note: This code path has nearly zero overhead when compared to
- * encodeLive().
- */
-
- return encodeLive(raba, buf).data();
-
- }
-
- public ICodedRaba encodeLive(final IRaba raba, final DataOutputBuffer buf) {
-
- if (raba == null)
- throw new UnsupportedOperationException();
-
- if (buf == null)
- throw new UnsupportedOperationException();
-
- final int n = raba.size();
-
- // This is sufficient capacity to code the data.
- final int initialCapacity = 1 + Bytes.SIZEOF_INT
- + BytesUtil.bitFlagByteLength(3 * n);
-
- buf.ensureCapacity(initialCapacity);
-
- // The byte offset of the start of the coded record in the buffer.
- final int O_origin = buf.pos();
-
- final int size = raba.size();
-
- buf.putByte(VERSION0);
-
- buf.putInt(size);
-
- /*
- * @todo use variant OBS(byte[], off, len) constructor and
- * pre-extend the buffer to have sufficient capacity so the OBS can
- * write directly onto the backing byte[], which will be much
- * faster.
- */
- final OutputBitStream obs = buf.getOutputBitStream();
-// final long O_values;
- try {
-
-// obs.writeInt(VERSION0, 8/* nbits */);
-//
-// obs.writeNibble(size);
-//
-// // Note: the bit offset where we start to code the values.
-// O_values = obs.writtenBits();
-
- for (int i = 0; i < size; i++) {
-
- if (raba.isNull(i)) {
-
- // flag a deleted value (de-serialize to a null).
- obs.writeInt(7, 3/* nbits */);
-
- } else {
-
- final byte[] val = raba.get(i);
-
- obs.writeInt((int) val[0], 3/* nbits */);
-
- }
-
- }
-
- // ALWAYS FLUSH.
- obs.flush();
-
- } catch (IOException ex) {
-
- throw new RuntimeException(ex);
-
- }
-
- // slice on just the coded data record.
- final AbstractFixedByteArrayBuffer slice = buf.slice(O_origin, buf.pos()
- - O_origin);
-
-// // adjusted bit offset to the start of the coded values in the slice.
-// final int O_valuesAdjusted = ((int) O_values) - O_origin << 3;
-//
-// return new CodedRabaImpl(slice, size, O_valuesAdjusted);
-
- return new CodedRabaImpl(slice, size);
-
- }
-
- public ICodedRaba decode(final AbstractFixedByteArrayBuffer data) {
-
- return new CodedRabaImpl(data);
-
- }
-
- /**
- * Decoder.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- */
- private static class CodedRabaImpl extends AbstractCodedRaba {
-
- private final AbstractFixedByteArrayBuffer data;
-
- /**
- * Cached.
- */
- private final int size;
-
-// /**
-// * Bit offset to the first coded value.
-// */
-// private final int O_values;
-
- final public AbstractFixedByteArrayBuffer data() {
-
- return data;
-
- }
-
- /**
- * No.
- */
- final public boolean isKeys() {
-
- return false;
-
- }
-
- final public int size() {
-
- return size;
-
- }
-
- final public int capacity() {
-
- return size;
-
- }
-
- final public boolean isEmpty() {
-
- return size == 0;
-
- }
-
- /**
- * Always <code>true</code>.
- */
- final public boolean isFull() {
-
- return true;
-
- }
-
- /**
- * Constructor used when encoding a data record.
- *
- * @param data
- * The coded data record.
- * @param size
- * The size of the coded {@link IRaba}.
- */
-// * @param O_values
-// * The bit offset of the start of the coded values.
- public CodedRabaImpl(final AbstractFixedByteArrayBuffer data,
- final int size) { //, final int O_values) {
-
- this.data = data;
-
- this.size = size;
-
-// this.O_values = O_values;
-
- }
-
- /**
- * Constructor used when decoding a data record.
- *
- * @param data
- * The coded data record.
- */
- public CodedRabaImpl(final AbstractFixedByteArrayBuffer data) {
-
- if (data == null)
- throw new IllegalArgumentException();
-
- this.data = data;
-
- final byte version = data.getByte(0/*off*/);
-
- if (version != VERSION0) {
-
- throw new RuntimeException("Unknown version=" + version);
-
- }
-
- size = data.getInt(1/*off*/);//ibs.readNibble();
-
-// final InputBitStream ibs = data.getInputBitStream();
-//
-// try {
-//
-// final byte version = (byte) ibs.readInt(8/* nbits */);
-//
-// if (version != VERSION0) {
-//
-// throw new RuntimeException("Unknown version=" + version);
-//
-// }
-//
-// size = ibs.readNibble();
-//
-// O_values = (int) ibs.readBits();
-//
-// } catch (IOException ex) {
-//
-// throw new RuntimeException(ex);
-//
-//// close not required for IBS backed by byte[] and has high overhead.
-//// } finally {
-////
-//// try {
-////
-//// ibs.close();
-////
-//// } catch (IOException ex) {
-////
-//// log.error(ex);
-////
-//// }
-////
-// }
-
- }
-
- /**
- * Thread-safe extract of the bits coded value for the specified index.
- *
- * @param index
- * The specified index.
- *
- * @return The bit coded value.
- *
- * @throws IndexOutOfBoundsException
- * unless the index is in [0:size-1].
- */
- final protected byte getBits(final int index) {
-
- if (index < 0 || index >= size)
- throw new IndexOutOfBoundsException();
-
- final InputBitStream ibs = data.getInputBitStream();
- try {
-
- ibs.position(O_values + (index * 3L));
-
- final int value = ibs.readInt(3/* nbits */);
-
- return (byte) (0xff & value);
-
- } catch(IOException ex) {
-
- throw new RuntimeException(ex);
-
-// close not required for IBS backed by byte[] and has high overhead.
-// } finally {
-// try {
-// ibs.close();
-// } catch(IOException ex) {
-// log.error(ex);
-// }
- }
-
-// int value = 0;
-//
-// for (int i = 0; i < 3; i++, bitIndex++) {
-//
-// final boolean bit = data.getBit(bitIndex);
-//
-// value |= (bit ? 1 : 0) << i;
-//
-// }
-//
-// return (byte) (value & 0xff);
-
- }
-
- final public int copy(final int index, final OutputStream os) {
-
- final byte bits = getBits(index);
-
- if (bits == 7) {
-
- // A null.
- throw new NullPointerException();
-
- } else {
-
- try {
-
- os.write(bits);
-
- } catch (IOException e) {
-
- throw new RuntimeException(e);
-
- }
-
- }
-
- return 1;
-
- }
-
- final public byte[] get(final int index) {
-
- final byte bits = getBits(index);
-
- if (bits == 7) {
-
- // A null.
- return null;
-
- } else {
-
- return new byte[] { bits };
-
- }
-
- }
-
- final public boolean isNull(final int index) {
-
- return getBits(index) == 7;
-
- }
-
- /**
- * Returns ONE (1) unless the value is a <code>null</code>.
- *
- * {@inheritDoc}
- */
- final public int length(final int index) {
-
- if (isNull(index))
- throw new NullPointerException();
-
- return 1;
-
- }
-
- /**
- * Not supported.
- */
- final public int search(final byte[] searchKey) {
-
- throw new UnsupportedOperationException();
-
- }
-
- }
-
-}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/InGraphBinarySearchFilter.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/InGraphBinarySearchFilter.java 2010-07-26 19:14:39 UTC (rev 3298)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/InGraphBinarySearchFilter.java 2010-07-26 19:31:59 UTC (rev 3299)
@@ -88,8 +88,28 @@
}
- public void readExternal(ObjectInput in) throws IOException,
+ /**
+ * The initial version.
+ */
+ private static final transient short VERSION0 = 0;
+
+ /**
+ * The current version.
+ */
+ private static final transient short VERSION = VERSION0;
+
+ public void readExternal(final ObjectInput in) throws IOException,
ClassNotFoundException {
+
+ final short version = in.readShort();
+
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new UnsupportedOperationException("Unknown version: "
+ + version);
+ }
final int size = in.readInt();
@@ -103,8 +123,10 @@
}
- public void writeExternal(ObjectOutput out) throws IOException {
+ public void writeExternal(final ObjectOutput out) throws IOException {
+ out.writeShort(VERSION);
+
out.writeInt(a.length);
for(IV iv : a) {
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/vocab/BaseVocabulary.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/vocab/BaseVocabulary.java 2010-07-26 19:14:39 UTC (rev 3298)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/vocab/BaseVocabulary.java 2010-07-26 19:31:59 UTC (rev 3299)
@@ -36,16 +36,17 @@
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
+
import org.apache.log4j.Logger;
import org.openrdf.model.Value;
import org.openrdf.model.ValueFactory;
import org.openrdf.model.impl.ValueFactoryImpl;
+
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.io.DataOutputBuffer;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.IVUtility;
-import com.bigdata.rdf.internal.TermId;
import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.model.BigdataValueFactory;
import com.bigdata.rdf.model.BigdataValueSerializer;
@@ -246,6 +247,16 @@
}
/**
+ * The initial version.
+ */
+ private static final transient short VERSION0 = 0;
+
+ /**
+ * The current version.
+ */
+ private static final transient short VERSION = VERSION0;
+
+ /**
* Note: The de-serialized state contains {@link Value}s but not
* {@link BigdataValue}s since the {@link AbstractTripleStore} reference is
* not available and we can not obtain the appropriate
@@ -258,6 +269,16 @@
if (values != null)
throw new IllegalStateException();
+ final short version = in.readShort();
+
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new UnsupportedOperationException("Unknown version: "
+ + version);
+ }
+
final ValueFactory valueFactory = new ValueFactoryImpl();
final BigdataValueSerializer<Value> valueSer = new BigdataValueSerializer<Value>(
@@ -305,6 +326,8 @@
if (values == null)
throw new IllegalStateException();
+
+ out.writeShort(VERSION);
final int nvalues = values.size();
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOValueCoders.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOValueCoders.java 2010-07-26 19:14:39 UTC (rev 3298)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOValueCoders.java 2010-07-26 19:31:59 UTC (rev 3299)
@@ -29,7 +29,9 @@
import java.io.IOException;
import java.util.Random;
+
import junit.framework.TestCase2;
+
import com.bigdata.btree.AbstractBTreeTestCase;
import com.bigdata.btree.ICounter;
import com.bigdata.btree.raba.IRaba;
@@ -40,7 +42,6 @@
import com.bigdata.io.AbstractFixedByteArrayBuffer;
import com.bigdata.io.DataOutputBuffer;
import com.bigdata.io.FixedByteArrayBuffer;
-import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.TermId;
import com.bigdata.rdf.internal.VTE;
import com.bigdata.rdf.lexicon.ITermIdCodes;
@@ -186,12 +187,12 @@
}
- public void test_FastRDFValueCoder() {
+// public void test_FastRDFValueCoder() {
+//
+// doRoundTripTests(new FastRDFValueCoder(), false/* sids */, true/* inference */);
+//
+// }
- doRoundTripTests(new FastRDFValueCoder(), false/* sids */, true/* inference */);
-
- }
-
/**
* Simple tests for {@link FastRDFValueCoder2}.
* <P>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2010-07-26 19:14:46
|
Revision: 3298
http://bigdata.svn.sourceforge.net/bigdata/?rev=3298&view=rev
Author: mrpersonick
Date: 2010-07-26 19:14:39 +0000 (Mon, 26 Jul 2010)
Log Message:
-----------
fixing bnodes, moving extensions to samples
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataValueFactoryImpl.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestInlining.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/TestTripleStore.java
Added Paths:
-----------
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/rdf/
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/rdf/internal/
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/rdf/internal/ColorsEnumExtension.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/rdf/internal/EpochExtension.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/rdf/internal/SampleExtensionFactory.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataValueFactoryImpl.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataValueFactoryImpl.java 2010-07-26 19:13:50 UTC (rev 3297)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataValueFactoryImpl.java 2010-07-26 19:14:39 UTC (rev 3298)
@@ -173,7 +173,7 @@
*/
protected String nextID() {
- return "_"+UUID.randomUUID();
+ return "u"+UUID.randomUUID();
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java 2010-07-26 19:13:50 UTC (rev 3297)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java 2010-07-26 19:14:39 UTC (rev 3298)
@@ -44,7 +44,6 @@
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.log4j.Logger;
import org.openrdf.model.Literal;
import org.openrdf.model.Resource;
@@ -56,7 +55,6 @@
import org.openrdf.model.vocabulary.RDFS;
import org.openrdf.model.vocabulary.XMLSchema;
import org.openrdf.rio.rdfxml.RDFXMLParser;
-
import com.bigdata.btree.AbstractBTree;
import com.bigdata.btree.BTree;
import com.bigdata.btree.BytesUtil;
@@ -80,6 +78,8 @@
import com.bigdata.rdf.inf.JustificationIterator;
import com.bigdata.rdf.internal.DefaultExtensionFactory;
import com.bigdata.rdf.internal.IDatatypeURIResolver;
+import com.bigdata.rdf.internal.IExtension;
+import com.bigdata.rdf.internal.IExtensionFactory;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.lexicon.BigdataRDFFullTextIndex;
import com.bigdata.rdf.lexicon.ITermIdCodes;
Copied: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/rdf/internal/ColorsEnumExtension.java (from rev 3289, branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java)
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/rdf/internal/ColorsEnumExtension.java (rev 0)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/rdf/internal/ColorsEnumExtension.java 2010-07-26 19:14:39 UTC (rev 3298)
@@ -0,0 +1,169 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package com.bigdata.rdf.internal;
+
+import org.openrdf.model.Literal;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.model.BigdataValueFactory;
+import com.bigdata.rdf.store.BD;
+
+/**
+ * Example of how to do a custom enum and map that enum over a byte using a
+ * native inline {@link XSDByteIV}.
+ */
+public class ColorsEnumExtension<V extends BigdataValue> implements IExtension<V> {
+
+ /**
+ * The datatype URI for the colors enum extension.
+ */
+ public transient static final URI COLOR = new URIImpl(BD.NAMESPACE + "Color");
+
+ private BigdataURI color;
+
+ public ColorsEnumExtension() {
+
+ }
+
+ public void resolveDatatype(final IDatatypeURIResolver resolver) {
+
+ this.color = resolver.resolve(COLOR);
+
+ }
+
+ public BigdataURI getDatatype() {
+
+ return color;
+
+ }
+
+ /**
+ * Attempts to convert the supplied RDF value into a colors enum
+ * representation. Tests for a literal value with the correct datatype
+ * that can be converted to one of the colors in the {@link Color} enum
+ * based on the string value of the literal's label. Each {@link Color}
+ * in the enum maps to a particular byte. This byte is encoded in a
+ * delegate {@link XSDByteIV}, and an {@link ExtensionIV} is returned that
+ * wraps the native type.
+ */
+ public ExtensionIV createIV(final Value value) {
+
+ if (value instanceof Literal == false)
+ throw new IllegalArgumentException();
+
+ final Literal l = (Literal) value;
+
+ if (l.getDatatype() == null || !color.equals(l.getDatatype()))
+ throw new IllegalArgumentException();
+
+ final String s = value.stringValue();
+
+ final Color c = Enum.valueOf(Color.class, s);
+
+ // not a valid color
+ if (c == null)
+ return null;
+
+ final AbstractLiteralIV delegate = new XSDByteIV(c.getByte());
+
+ return new ExtensionIV(delegate, (TermId) getDatatype().getIV());
+
+ }
+
+ /**
+ * Attempt to convert the {@link AbstractLiteralIV#byteValue()} back into
+ * a {@link Color}, and then use the string value of the {@link Color} to
+ * create an RDF literal.
+ */
+ public V asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
+
+ final byte b = iv.getDelegate().byteValue();
+
+ final Color c = Color.valueOf(b);
+
+ if (c == null)
+ throw new RuntimeException("bad color got encoded somehow");
+
+ return (V) vf.createLiteral(c.toString(), color);
+
+ }
+
+ /**
+ * Simple demonstration enum for some common colors. Can fit up to 256 enum
+ * values into an enum projected onto a byte.
+ */
+ public enum Color {
+
+ Red((byte) 0),
+ Blue((byte) 1),
+ Green((byte) 2),
+ Yellow((byte) 3),
+ Orange((byte) 4),
+ Purple((byte) 5),
+ Black((byte) 6),
+ White((byte) 7),
+ Brown((byte) 8);
+
+ private Color(final byte b) {
+ this.b = b;
+ }
+
+ static final public Color valueOf(final byte b) {
+ switch (b) {
+ case 0:
+ return Red;
+ case 1:
+ return Blue;
+ case 2:
+ return Green;
+ case 3:
+ return Yellow;
+ case 4:
+ return Orange;
+ case 5:
+ return Purple;
+ case 6:
+ return Black;
+ case 7:
+ return White;
+ case 8:
+ return Brown;
+ default:
+ throw new IllegalArgumentException(Byte.toString(b));
+ }
+ }
+
+ private final byte b;
+
+ public byte getByte() {
+ return b;
+ }
+
+ }
+
+}
Copied: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/rdf/internal/EpochExtension.java (from rev 3289, branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java)
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/rdf/internal/EpochExtension.java (rev 0)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/rdf/internal/EpochExtension.java 2010-07-26 19:14:39 UTC (rev 3298)
@@ -0,0 +1,110 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package com.bigdata.rdf.internal;
+
+import org.openrdf.model.Literal;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.datatypes.XMLDatatypeUtil;
+import org.openrdf.model.impl.URIImpl;
+import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.model.BigdataValueFactory;
+import com.bigdata.rdf.store.BD;
+
+/**
+ * This implementation of {@link IExtension} implements inlining for literals
+ * that represent time in milliseconds since the epoch. The milliseconds are
+ * encoded as an inline long.
+ */
+public class EpochExtension<V extends BigdataValue> implements IExtension<V> {
+
+ /**
+ * The datatype URI for the epoch extension.
+ */
+ public transient static final URI EPOCH = new URIImpl(BD.NAMESPACE + "Epoch");
+
+ private BigdataURI epoch;
+
+ public EpochExtension() {
+ }
+
+ public void resolveDatatype(final IDatatypeURIResolver resolver) {
+
+ this.epoch = resolver.resolve(EPOCH);
+
+ }
+
+ public BigdataURI getDatatype() {
+
+ return epoch;
+
+ }
+
+ /**
+ * Attempts to convert the supplied value into an epoch representation.
+ * Tests for a literal value with the correct datatype that can be converted
+ * to a positive long integer. Encodes the long in a delegate
+ * {@link XSDLongIV}, and returns an {@link ExtensionIV} to wrap the native
+ * type.
+ */
+ public ExtensionIV createIV(final Value value) {
+
+ if (value instanceof Literal == false)
+ throw new IllegalArgumentException();
+
+ final Literal lit = (Literal) value;
+
+ final URI dt = lit.getDatatype();
+
+ if (dt == null || !EPOCH.stringValue().equals(dt.stringValue()))
+ throw new IllegalArgumentException();
+
+ final String s = value.stringValue();
+
+ final long l = XMLDatatypeUtil.parseLong(s);
+
+ // can't have negative epoch values
+ if (l < 0)
+ return null;
+
+ final AbstractLiteralIV delegate = new XSDLongIV(l);
+
+ return new ExtensionIV(delegate, (TermId) getDatatype().getIV());
+
+ }
+
+ /**
+ * Use the string value of the {@link ExtensionIV} (which defers to the
+ * string value of the native type) to create a literal with the epoch
+ * datatype.
+ */
+ public V asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
+
+ return (V) vf.createLiteral(iv.stringValue(), epoch);
+
+ }
+
+}
Added: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/rdf/internal/SampleExtensionFactory.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/rdf/internal/SampleExtensionFactory.java (rev 0)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/samples/com/bigdata/rdf/internal/SampleExtensionFactory.java 2010-07-26 19:14:39 UTC (rev 3298)
@@ -0,0 +1,35 @@
+package com.bigdata.rdf.internal;
+
+/**
+ * Simple {@link IExtensionFactory} implementation that creates two
+ * {@link IExtension}s - the {@link EpochExtension} and the
+ * {@link ColorsEnumExtension}.
+ */
+public class SampleExtensionFactory implements IExtensionFactory {
+
+ private final IExtension[] extensions;
+
+ public SampleExtensionFactory() {
+
+ extensions = new IExtension[] {
+ new EpochExtension(),
+ new ColorsEnumExtension()
+ };
+
+ }
+
+ public void resolveDatatypes(final IDatatypeURIResolver resolver) {
+
+ for (IExtension extension : extensions) {
+ extension.resolveDatatype(resolver);
+ }
+
+ }
+
+ public IExtension[] getExtensions() {
+
+ return extensions;
+
+ }
+
+}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestInlining.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestInlining.java 2010-07-26 19:13:50 UTC (rev 3297)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestInlining.java 2010-07-26 19:14:39 UTC (rev 3298)
@@ -38,6 +38,7 @@
import com.bigdata.rdf.internal.EpochExtension;
import com.bigdata.rdf.internal.ExtensionIV;
import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.internal.SampleExtensionFactory;
import com.bigdata.rdf.internal.XSD;
import com.bigdata.rdf.model.BigdataBNode;
import com.bigdata.rdf.model.BigdataLiteral;
@@ -195,17 +196,19 @@
// lookup/add some values.
final BigdataValueFactory f = store.getValueFactory();
- final BigdataBNode b1 = f.createBNode("1");
- final BigdataBNode b01 = f.createBNode("01");
- final BigdataBNode b2 = f.createBNode(UUID.randomUUID().toString());
+ final BigdataBNode b1 = f.createBNode("i1");
+ final BigdataBNode b01 = f.createBNode("i01");
+ final BigdataBNode b2 = f.createBNode("u"+UUID.randomUUID().toString());
final BigdataBNode b3 = f.createBNode("foo");
final BigdataBNode b4 = f.createBNode("foo12345");
+ final BigdataBNode b5 = f.createBNode("12345");
terms.add(b1);
terms.add(b01);
terms.add(b2);
terms.add(b3);
terms.add(b4);
+ terms.add(b5);
final Map<IV, BigdataValue> ids = doAddTermsTest(store, terms);
@@ -214,6 +217,7 @@
assertTrue(b2.getIV().isInline());
assertFalse(b3.getIV().isInline());
assertFalse(b4.getIV().isInline());
+ assertFalse(b5.getIV().isInline());
if (store.isStable()) {
@@ -261,6 +265,10 @@
// test w/o the full text index.
properties.setProperty(Options.TEXT_INDEX, "false");
+ // test with the sample extension factory
+ properties.setProperty(Options.EXTENSION_FACTORY_CLASS,
+ SampleExtensionFactory.class.getName());
+
AbstractTripleStore store = getStore(properties);
try {
@@ -349,6 +357,10 @@
// test w/o the full text index.
properties.setProperty(Options.TEXT_INDEX, "false");
+ // test with the sample extension factory
+ properties.setProperty(Options.EXTENSION_FACTORY_CLASS,
+ SampleExtensionFactory.class.getName());
+
AbstractTripleStore store = getStore(properties);
try {
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/TestTripleStore.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/TestTripleStore.java 2010-07-26 19:13:50 UTC (rev 3297)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/TestTripleStore.java 2010-07-26 19:14:39 UTC (rev 3298)
@@ -586,9 +586,12 @@
final IV bn1_id = store.addTerm(bn1);
assertTrue(bn1_id != NULL);
+ System.err.println(bn1_id);
final IV bn2_id = store.addTerm(bn2);
assertTrue(bn2_id != NULL);
-
+ System.err.println(bn2_id);
+ System.err.println(storeBlankNodes);
+
assertEquals("#terms", 8 + 3 + (store.getLexiconRelation()
.isStoreBlankNodes() ? 2 : 0), store.getTermCount());
assertEquals("#uris", 8, store.getURICount());
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2010-07-26 19:13:59
|
Revision: 3297
http://bigdata.svn.sourceforge.net/bigdata/?rev=3297&view=rev
Author: mrpersonick
Date: 2010-07-26 19:13:50 +0000 (Mon, 26 Jul 2010)
Log Message:
-----------
fixing bnodes, moving extensions to samples
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/.classpath
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/NumericBNodeIV.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/UUIDBNodeIV.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
Removed Paths:
-------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java
Modified: branches/LEXICON_REFACTOR_BRANCH/.classpath
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/.classpath 2010-07-26 18:16:07 UTC (rev 3296)
+++ branches/LEXICON_REFACTOR_BRANCH/.classpath 2010-07-26 19:13:50 UTC (rev 3297)
@@ -1,6 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry path="bigdata-rdf/src/java" kind="src"/>
+ <classpathentry path="bigdata-rdf/src/samples" kind="src"/>
<classpathentry path="bigdata-bsbm/src/test" kind="src"/>
<classpathentry path="dsi-utils/src/java" kind="src"/>
<classpathentry path="bigdata-bsbm/src/java" kind="src"/>
Deleted: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java 2010-07-26 18:16:07 UTC (rev 3296)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java 2010-07-26 19:13:50 UTC (rev 3297)
@@ -1,169 +0,0 @@
-/**
-
-Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-
-package com.bigdata.rdf.internal;
-
-import org.openrdf.model.Literal;
-import org.openrdf.model.URI;
-import org.openrdf.model.Value;
-import org.openrdf.model.impl.URIImpl;
-import com.bigdata.rdf.model.BigdataURI;
-import com.bigdata.rdf.model.BigdataValue;
-import com.bigdata.rdf.model.BigdataValueFactory;
-import com.bigdata.rdf.store.BD;
-
-/**
- * Example of how to do a custom enum and map that enum over a byte using a
- * native inline {@link XSDByteIV}.
- */
-public class ColorsEnumExtension<V extends BigdataValue> implements IExtension<V> {
-
- /**
- * The datatype URI for the colors enum extension.
- */
- public static final URI COLOR = new URIImpl(BD.NAMESPACE + "Color");
-
- private BigdataURI color;
-
- public ColorsEnumExtension() {
-
- }
-
- public void resolveDatatype(final IDatatypeURIResolver resolver) {
-
- this.color = resolver.resolve(COLOR);
-
- }
-
- public BigdataURI getDatatype() {
-
- return color;
-
- }
-
- /**
- * Attempts to convert the supplied RDF value into a colors enum
- * representation. Tests for a literal value with the correct datatype
- * that can be converted to one of the colors in the {@link Color} enum
- * based on the string value of the literal's label. Each {@link Color}
- * in the enum maps to a particular byte. This byte is encoded in a
- * delegate {@link XSDByteIV}, and an {@link ExtensionIV} is returned that
- * wraps the native type.
- */
- public ExtensionIV createIV(final Value value) {
-
- if (value instanceof Literal == false)
- throw new IllegalArgumentException();
-
- final Literal l = (Literal) value;
-
- if (l.getDatatype() == null || !color.equals(l.getDatatype()))
- throw new IllegalArgumentException();
-
- final String s = value.stringValue();
-
- final Color c = Enum.valueOf(Color.class, s);
-
- // not a valid color
- if (c == null)
- return null;
-
- final AbstractLiteralIV delegate = new XSDByteIV(c.getByte());
-
- return new ExtensionIV(delegate, (TermId) getDatatype().getIV());
-
- }
-
- /**
- * Attempt to convert the {@link AbstractLiteralIV#byteValue()} back into
- * a {@link Color}, and then use the string value of the {@link Color} to
- * create an RDF literal.
- */
- public V asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
-
- final byte b = iv.getDelegate().byteValue();
-
- final Color c = Color.valueOf(b);
-
- if (c == null)
- throw new RuntimeException("bad color got encoded somehow");
-
- return (V) vf.createLiteral(c.toString(), color);
-
- }
-
- /**
- * Simple demonstration enum for some common colors. Can fit up to 256 enum
- * values into an enum projected onto a byte.
- */
- public enum Color {
-
- Red((byte) 0),
- Blue((byte) 1),
- Green((byte) 2),
- Yellow((byte) 3),
- Orange((byte) 4),
- Purple((byte) 5),
- Black((byte) 6),
- White((byte) 7),
- Brown((byte) 8);
-
- private Color(final byte b) {
- this.b = b;
- }
-
- static final public Color valueOf(final byte b) {
- switch (b) {
- case 0:
- return Red;
- case 1:
- return Blue;
- case 2:
- return Green;
- case 3:
- return Yellow;
- case 4:
- return Orange;
- case 5:
- return Purple;
- case 6:
- return Black;
- case 7:
- return White;
- case 8:
- return Brown;
- default:
- throw new IllegalArgumentException(Byte.toString(b));
- }
- }
-
- private final byte b;
-
- public byte getByte() {
- return b;
- }
-
- }
-
-}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java 2010-07-26 18:16:07 UTC (rev 3296)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java 2010-07-26 19:13:50 UTC (rev 3297)
@@ -1,9 +1,7 @@
package com.bigdata.rdf.internal;
/**
- * Simple {@link IExtensionFactory} implementation that creates two
- * {@link IExtension}s - the {@link EpochExtension} and the
- * {@link ColorsEnumExtension}.
+ * Empty {@link IExtensionFactory}.
*/
public class DefaultExtensionFactory implements IExtensionFactory {
@@ -12,8 +10,6 @@
public DefaultExtensionFactory() {
extensions = new IExtension[] {
- new EpochExtension(),
- new ColorsEnumExtension()
};
}
Deleted: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java 2010-07-26 18:16:07 UTC (rev 3296)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java 2010-07-26 19:13:50 UTC (rev 3297)
@@ -1,110 +0,0 @@
-/**
-
-Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-
-package com.bigdata.rdf.internal;
-
-import org.openrdf.model.Literal;
-import org.openrdf.model.URI;
-import org.openrdf.model.Value;
-import org.openrdf.model.datatypes.XMLDatatypeUtil;
-import org.openrdf.model.impl.URIImpl;
-import com.bigdata.rdf.model.BigdataURI;
-import com.bigdata.rdf.model.BigdataValue;
-import com.bigdata.rdf.model.BigdataValueFactory;
-import com.bigdata.rdf.store.BD;
-
-/**
- * This implementation of {@link IExtension} implements inlining for literals
- * that represent time in milliseconds since the epoch. The milliseconds are
- * encoded as an inline long.
- */
-public class EpochExtension<V extends BigdataValue> implements IExtension<V> {
-
- /**
- * The datatype URI for the epoch extension.
- */
- public static final transient URI EPOCH = new URIImpl(BD.NAMESPACE + "Epoch");
-
- private BigdataURI epoch;
-
- public EpochExtension() {
- }
-
- public void resolveDatatype(final IDatatypeURIResolver resolver) {
-
- this.epoch = resolver.resolve(EPOCH);
-
- }
-
- public BigdataURI getDatatype() {
-
- return epoch;
-
- }
-
- /**
- * Attempts to convert the supplied value into an epoch representation.
- * Tests for a literal value with the correct datatype that can be converted
- * to a positive long integer. Encodes the long in a delegate
- * {@link XSDLongIV}, and returns an {@link ExtensionIV} to wrap the native
- * type.
- */
- public ExtensionIV createIV(final Value value) {
-
- if (value instanceof Literal == false)
- throw new IllegalArgumentException();
-
- final Literal lit = (Literal) value;
-
- final URI dt = lit.getDatatype();
-
- if (dt == null || !EPOCH.stringValue().equals(dt.stringValue()))
- throw new IllegalArgumentException();
-
- final String s = value.stringValue();
-
- final long l = XMLDatatypeUtil.parseLong(s);
-
- // can't have negative epoch values
- if (l < 0)
- return null;
-
- final AbstractLiteralIV delegate = new XSDLongIV(l);
-
- return new ExtensionIV(delegate, (TermId) getDatatype().getIV());
-
- }
-
- /**
- * Use the string value of the {@link ExtensionIV} (which defers to the
- * string value of the native type) to create a literal with the epoch
- * datatype.
- */
- public V asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
-
- return (V) vf.createLiteral(iv.stringValue(), epoch);
-
- }
-
-}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java 2010-07-26 18:16:07 UTC (rev 3296)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java 2010-07-26 19:13:50 UTC (rev 3297)
@@ -27,7 +27,7 @@
/**
* IExtensionFactories are responsible for enumerating what extensions are
* supported for a particular database configuration. Bigdata comes packaged
- * with a {@link DefaultExtensionFactory} that supplies two starter extensions -
+ * with a {@link SampleExtensionFactory} that supplies two starter extensions -
* the {@link EpochExtension} (for representing time since the epoch as a long
* integer) and the {@link ColorsEnumExtension} (a sample extension for how to
* represent an enumeration via inline literals).
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java 2010-07-26 18:16:07 UTC (rev 3296)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java 2010-07-26 19:13:50 UTC (rev 3297)
@@ -20,7 +20,7 @@
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
+ */
/*
* Created July 10, 2010
*/
@@ -39,10 +39,8 @@
import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.model.BigdataValueFactory;
-
/**
- * An object which describes which kinds of RDF Values are inlined into the
- * statement indices and how other RDF Values are coded into the lexicon.
+ * An object which describes which kinds of RDF Values are inlined into the statement indices and how other RDF Values are coded into the lexicon.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
@@ -50,21 +48,18 @@
* @todo large literal size boundary.
* @todo other configuration options.
*/
-public class LexiconConfiguration<V extends BigdataValue>
- implements ILexiconConfiguration {
-
+public class LexiconConfiguration<V extends BigdataValue> implements ILexiconConfiguration {
+
private boolean inlineLiterals, inlineBNodes;
-
+
private Map<TermId, IExtension> termIds;
-
+
private Map<String, IExtension> datatypes;
-
- public LexiconConfiguration(final boolean inlineLiterals,
- final boolean inlineBNodes,
- final IExtensionFactory xFactory) {
+
+ public LexiconConfiguration(final boolean inlineLiterals, final boolean inlineBNodes, final IExtensionFactory xFactory) {
this.inlineLiterals = inlineLiterals;
this.inlineBNodes = inlineBNodes;
-
+
termIds = new HashMap<TermId, IExtension>();
datatypes = new HashMap<String, IExtension>();
for (IExtension extension : xFactory.getExtensions()) {
@@ -74,59 +69,58 @@
termIds.put((TermId) datatype.getIV(), extension);
datatypes.put(datatype.stringValue(), extension);
}
-
+
}
-
+
public V asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
final TermId datatype = iv.getExtensionDatatype();
return (V) termIds.get(datatype).asValue(iv, vf);
}
-
+
public IV createInlineIV(final Value value) {
// we know right away we can't handle URIs
if (value instanceof URI)
return null;
-
+
if (value instanceof Literal) {
-
+
final Literal l = (Literal) value;
-
+
final URI datatype = l.getDatatype();
-
+
// not a datatyped literal
if (datatype == null)
return null;
if (datatypes.containsKey(datatype.stringValue())) {
-
+
try {
-
- final IExtension xFactory =
- datatypes.get(datatype.stringValue());
-
+
+ final IExtension xFactory = datatypes.get(datatype.stringValue());
+
final IV iv = xFactory.createIV(value);
-
+
if (iv != null && value instanceof BigdataValue)
((BigdataValue) value).setIV(iv);
-
+
return iv;
-
+
} catch (Exception ex) {
-
+
/*
* Some sort of parse error in the literal value most
* likely. Resort to term identifiers.
*/
return null;
-
+
}
-
+
}
-
+
// get the native DTE
final DTE dte = DTE.valueOf(datatype);
-
+
// no native DTE for this datatype
if (dte == null)
return null;
@@ -134,159 +128,165 @@
// check to see if we are inlining literals of this type
if (!isInline(VTE.LITERAL, dte))
return null;
-
+
final String v = value.stringValue();
-
+
IV iv = null;
-
+
try {
-
- switch(dte) {
- case XSDBoolean:
- iv = new XSDBooleanIV(XMLDatatypeUtil.parseBoolean(v));
- break;
- case XSDByte:
- iv = new XSDByteIV(XMLDatatypeUtil.parseByte(v));
- break;
- case XSDShort:
- iv = new XSDShortIV(XMLDatatypeUtil.parseShort(v));
- break;
- case XSDInt:
- iv = new XSDIntIV(XMLDatatypeUtil.parseInt(v));
- break;
- case XSDLong:
- iv = new XSDLongIV(XMLDatatypeUtil.parseLong(v));
- break;
- case XSDFloat:
- iv = new XSDFloatIV(XMLDatatypeUtil.parseFloat(v));
- break;
- case XSDDouble:
- iv = new XSDDoubleIV(XMLDatatypeUtil.parseDouble(v));
- break;
- case XSDInteger:
- iv = new XSDIntegerIV(XMLDatatypeUtil.parseInteger(v));
- break;
- case XSDDecimal:
- iv = new XSDDecimalIV(XMLDatatypeUtil.parseDecimal(v));
- break;
- case UUID:
- iv = new UUIDLiteralIV(UUID.fromString(v));
- break;
- default:
- iv = null;
+
+ switch (dte) {
+ case XSDBoolean:
+ iv = new XSDBooleanIV(XMLDatatypeUtil.parseBoolean(v));
+ break;
+ case XSDByte:
+ iv = new XSDByteIV(XMLDatatypeUtil.parseByte(v));
+ break;
+ case XSDShort:
+ iv = new XSDShortIV(XMLDatatypeUtil.parseShort(v));
+ break;
+ case XSDInt:
+ iv = new XSDIntIV(XMLDatatypeUtil.parseInt(v));
+ break;
+ case XSDLong:
+ iv = new XSDLongIV(XMLDatatypeUtil.parseLong(v));
+ break;
+ case XSDFloat:
+ iv = new XSDFloatIV(XMLDatatypeUtil.parseFloat(v));
+ break;
+ case XSDDouble:
+ iv = new XSDDoubleIV(XMLDatatypeUtil.parseDouble(v));
+ break;
+ case XSDInteger:
+ iv = new XSDIntegerIV(XMLDatatypeUtil.parseInteger(v));
+ break;
+ case XSDDecimal:
+ iv = new XSDDecimalIV(XMLDatatypeUtil.parseDecimal(v));
+ break;
+ case UUID:
+ iv = new UUIDLiteralIV(UUID.fromString(v));
+ break;
+ default:
+ iv = null;
}
-
+
} catch (NumberFormatException ex) {
-
+
// some dummy doesn't know how to format a number
// default to term identifier for this term
-
+
}
-
+
if (iv != null && value instanceof BigdataValue)
((BigdataValue) value).setIV(iv);
-
+
return iv;
-
+
} else if (value instanceof BNode) {
-
+
final BNode b = (BNode) value;
-
+
final String id = b.getID();
-
+
final char c = id.charAt(0);
- try {
+ if (c == 'u') {
+
+ try {
- final UUID uuid = UUID.fromString(id);
+ final UUID uuid = UUID.fromString(id.substring(1));
+
+ if (!uuid.toString().equals(id.substring(1)))
+ return null;
+
+ if (!isInline(VTE.BNODE, DTE.UUID))
+ return null;
+
+ final IV iv = new UUIDBNodeIV(uuid);
+
+ if (value instanceof BigdataValue)
+ ((BigdataValue) value).setIV(iv);
+
+ return iv;
+
+ } catch (Exception ex) {
+
+ // string id could not be converted to a UUID
+
+ }
- if (!uuid.toString().equals(id))
- return null;
+ } else if (c == 'i') {
- if (!isInline(VTE.BNODE, DTE.UUID))
- return null;
-
- final IV iv = new UUIDBNodeIV(uuid);
-
- if (value instanceof BigdataValue)
- ((BigdataValue) value).setIV(iv);
-
- return iv;
-
- } catch (Exception ex) {
-
- // string id could not be converted to a UUID
-
- }
-
- try {
+ try {
- final Integer i = Integer.valueOf(id);
+ final Integer i = Integer.valueOf(id.substring(1));
+
+ // cannot normalize id, needs to remain syntactically identical
+ if (!i.toString().equals(id.substring(1)))
+ return null;
+
+ if (!isInline(VTE.BNODE, DTE.XSDInt))
+ return null;
+
+ final IV iv = new NumericBNodeIV(i);
+
+ if (value instanceof BigdataValue)
+ ((BigdataValue) value).setIV(iv);
+
+ return iv;
+
+ } catch (Exception ex) {
+
+ // string id could not be converted to an Integer
+
+ }
- // cannot normalize id, needs to remain syntactically identical
- if (!i.toString().equals(id))
- return null;
-
- if (!isInline(VTE.BNODE, DTE.XSDInt))
- return null;
-
- final IV iv = new NumericBNodeIV(i);
-
- if (value instanceof BigdataValue)
- ((BigdataValue) value).setIV(iv);
-
- return iv;
-
- } catch (Exception ex) {
-
- // string id could not be converted to an Integer
-
}
}
-
+
return null;
-
+
}
-
+
/**
* See {@link ILexiconConfiguration#isInline(VTE, DTE)}.
*/
public boolean isInline(final VTE vte, final DTE dte) {
-
- switch(vte) {
- case BNODE:
- return inlineBNodes && isSupported(dte);
- case LITERAL:
- return inlineLiterals && isSupported(dte);
- default:
- return false;
+
+ switch (vte) {
+ case BNODE:
+ return inlineBNodes && isSupported(dte);
+ case LITERAL:
+ return inlineLiterals && isSupported(dte);
+ default:
+ return false;
}
-
+
}
-
+
private boolean isSupported(DTE dte) {
-
+
switch (dte) {
- case XSDBoolean:
- case XSDByte:
- case XSDShort:
- case XSDInt:
- case XSDLong:
- case XSDFloat:
- case XSDDouble:
- case XSDInteger:
- case XSDDecimal:
- case UUID:
- return true;
- case XSDUnsignedByte: // none of the unsigneds are tested yet
- case XSDUnsignedShort: // none of the unsigneds are tested yet
- case XSDUnsignedInt: // none of the unsigneds are tested yet
- case XSDUnsignedLong: // none of the unsigneds are tested yet
- default:
- return false;
+ case XSDBoolean:
+ case XSDByte:
+ case XSDShort:
+ case XSDInt:
+ case XSDLong:
+ case XSDFloat:
+ case XSDDouble:
+ case XSDInteger:
+ case XSDDecimal:
+ case UUID:
+ return true;
+ case XSDUnsignedByte: // none of the unsigneds are tested yet
+ case XSDUnsignedShort: // none of the unsigneds are tested yet
+ case XSDUnsignedInt: // none of the unsigneds are tested yet
+ case XSDUnsignedLong: // none of the unsigneds are tested yet
+ default:
+ return false;
}
-
+
}
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/NumericBNodeIV.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/NumericBNodeIV.java 2010-07-26 18:16:07 UTC (rev 3296)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/NumericBNodeIV.java 2010-07-26 19:13:50 UTC (rev 3297)
@@ -60,7 +60,7 @@
@Override
public String stringValue() {
- return String.valueOf(id);
+ return 'i'+String.valueOf(id);
}
final public Integer getInlineValue() {
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/UUIDBNodeIV.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/UUIDBNodeIV.java 2010-07-26 18:16:07 UTC (rev 3296)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/UUIDBNodeIV.java 2010-07-26 19:13:50 UTC (rev 3297)
@@ -64,7 +64,7 @@
@Override
public String stringValue() {
- return id.toString();
+ return 'u'+id.toString();
}
final public UUID getInlineValue() {
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-07-26 18:16:07 UTC (rev 3296)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-07-26 19:13:50 UTC (rev 3297)
@@ -75,9 +75,6 @@
import com.bigdata.journal.IIndexManager;
import com.bigdata.journal.IResourceLock;
import com.bigdata.rawstore.Bytes;
-import com.bigdata.rdf.internal.ColorsEnumExtension;
-import com.bigdata.rdf.internal.DefaultExtensionFactory;
-import com.bigdata.rdf.internal.EpochExtension;
import com.bigdata.rdf.internal.IExtension;
import com.bigdata.rdf.internal.IDatatypeURIResolver;
import com.bigdata.rdf.internal.IExtensionFactory;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-07-26 18:16:20
|
Revision: 3296
http://bigdata.svn.sourceforge.net/bigdata/?rev=3296&view=rev
Author: thompsonbry
Date: 2010-07-26 18:16:07 +0000 (Mon, 26 Jul 2010)
Log Message:
-----------
typo broke the code in writeExternal.
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/DefaultTupleSerializer.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/DefaultTupleSerializer.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/DefaultTupleSerializer.java 2010-07-26 18:15:10 UTC (rev 3295)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/DefaultTupleSerializer.java 2010-07-26 18:16:07 UTC (rev 3296)
@@ -340,7 +340,7 @@
}
- public void writeExternal(final bjectOutput out) throws IOException {
+ public void writeExternal(final ObjectOutput out) throws IOException {
out.writeShort(VERSION);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-07-26 18:15:16
|
Revision: 3295
http://bigdata.svn.sourceforge.net/bigdata/?rev=3295&view=rev
Author: thompsonbry
Date: 2010-07-26 18:15:10 +0000 (Mon, 26 Jul 2010)
Log Message:
-----------
Added / simplified explicit versioning.
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/DefaultTupleSerializer.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Id2TermTupleSerializer.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/DefaultTupleSerializer.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/DefaultTupleSerializer.java 2010-07-26 18:14:42 UTC (rev 3294)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/DefaultTupleSerializer.java 2010-07-26 18:15:10 UTC (rev 3295)
@@ -308,20 +308,42 @@
}
- public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ /**
+ * The initial version.
+ * <p>
+ * Note: Explicit versioning for the {@link DefaultTupleSerializer} was
+ * introduced with inlining of datatype literals for the RDF database.
+ */
+ private final static transient int VERSION0 = 0;
- delegateKeyBuilderFactory = (IKeyBuilderFactory)in.readObject();
-
- threadLocalKeyBuilderFactory = new ThreadLocalKeyBuilderFactory(delegateKeyBuilderFactory);
-
- leafKeysCoder = (IRabaCoder) in.readObject();
+ /**
+ * The current version.
+ */
+ private final static transient int VERSION = VERSION0;
- leafValsCoder = (IRabaCoder) in.readObject();
+ public void readExternal(final ObjectInput in) throws IOException,
+ ClassNotFoundException {
+ final short version = in.readShort();
+ switch (version) {
+ case VERSION0:
+ delegateKeyBuilderFactory = (IKeyBuilderFactory) in.readObject();
+ threadLocalKeyBuilderFactory = new ThreadLocalKeyBuilderFactory(
+ delegateKeyBuilderFactory);
+ leafKeysCoder = (IRabaCoder) in.readObject();
+ leafValsCoder = (IRabaCoder) in.readObject();
+ break;
+ default:
+ throw new UnsupportedOperationException("Unknown version: "
+ + version);
+ }
+
}
- public void writeExternal(ObjectOutput out) throws IOException {
+ public void writeExternal(final bjectOutput out) throws IOException {
+ out.writeShort(VERSION);
+
out.writeObject(delegateKeyBuilderFactory);
out.writeObject(leafKeysCoder);
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Id2TermTupleSerializer.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Id2TermTupleSerializer.java 2010-07-26 18:14:42 UTC (rev 3294)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Id2TermTupleSerializer.java 2010-07-26 18:15:10 UTC (rev 3295)
@@ -32,7 +32,9 @@
import java.io.ObjectOutput;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+
import org.openrdf.model.Value;
+
import com.bigdata.btree.DefaultTupleSerializer;
import com.bigdata.btree.ITuple;
import com.bigdata.btree.keys.ASCIIKeyBuilderFactory;
@@ -202,55 +204,30 @@
return tmp;
}
-
+
/**
- * Included only the UTF serialization of the namespace field without
- * explicit version support.
- *
* <pre>
+ * valueFactoryClass:UTF
* namespace:UTF
* </pre>
*/
static final transient short VERSION0 = 0;
- /**
- * Added the UTF serialization of the class name of the value factory
- * and an explicit version number in the serialization format. This
- * version is detected by a read of an empty string from the original
- * UTF field.
- *
- * <pre>
- * "":UTF
- * valueFactoryClass:UTF
- * namespace:UTF
- * </pre>
- *
- * Note: THere are unit tests for this backward compatible serialization
- * change in TestId2TermTupleSerializer.
- */
- static final transient short VERSION1 = 1;
+ private static final transient short VERSION = VERSION0;
- private static final transient short VERSION = VERSION1;
-
- public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ public void readExternal(final ObjectInput in) throws IOException, ClassNotFoundException {
super.readExternal(in);
- short version = VERSION0;
- String s1 = in.readUTF();
- String s2 = BigdataValueFactoryImpl.class.getName();
- if (s1.length() == 0) {
- version = in.readShort();
- s1 = in.readUTF();
- s2 = in.readUTF();
- }
+ final short version = in.readShort();
+ final String namespace;
+ final String valueFactoryClass;
switch (version) {
case VERSION0:
- case VERSION1:
+ namespace = in.readUTF();
+ valueFactoryClass = in.readUTF();
break;
default:
throw new IOException("unknown version=" + version);
}
- final String namespace = s1;
- final String valueFactoryClass = s2;
// set the namespace field.
this.namespace = namespace;
// resolve the valueSerializer from the value factory class.
@@ -275,18 +252,14 @@
valueSer = this.valueFactory.getValueSerializer();
}
- public void writeExternal(ObjectOutput out) throws IOException {
+ public void writeExternal(final ObjectOutput out) throws IOException {
super.writeExternal(out);
final short version = VERSION;
final String valueFactoryClass = valueFactory.getClass().getName();
+ out.writeShort(version);
switch (version) {
case VERSION0:
out.writeUTF(namespace);
- break;
- case VERSION1:
- out.writeUTF("");
- out.writeShort(version);
- out.writeUTF(namespace);
out.writeUTF(valueFactoryClass);
break;
default:
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-07-26 18:14:54
|
Revision: 3294
http://bigdata.svn.sourceforge.net/bigdata/?rev=3294&view=rev
Author: thompsonbry
Date: 2010-07-26 18:14:42 +0000 (Mon, 26 Jul 2010)
Log Message:
-----------
Made field transient
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java 2010-07-26 17:19:38 UTC (rev 3293)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java 2010-07-26 18:14:42 UTC (rev 3294)
@@ -44,7 +44,7 @@
/**
* The datatype URI for the epoch extension.
*/
- public static final URI EPOCH = new URIImpl(BD.NAMESPACE + "Epoch");
+ public static final transient URI EPOCH = new URIImpl(BD.NAMESPACE + "Epoch");
private BigdataURI epoch;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2010-07-26 17:19:44
|
Revision: 3293
http://bigdata.svn.sourceforge.net/bigdata/?rev=3293&view=rev
Author: martyncutcher
Date: 2010-07-26 17:19:38 +0000 (Mon, 26 Jul 2010)
Log Message:
-----------
Add further zero oriented tests and comment out invalid test data from bigDecimal0
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java 2010-07-26 17:17:51 UTC (rev 3292)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java 2010-07-26 17:19:38 UTC (rev 3293)
@@ -1634,7 +1634,7 @@
}
- if (cause != null || !expected.equals(actual)) {
+ if (cause != null || !(expected.compareTo(actual) == 0)) {
final String msg = "BigDecimal" + //
"\nexpected=" + expected + //
@@ -1690,6 +1690,32 @@
}
+ protected void doEQTest(final BigDecimal i1, final BigDecimal i2) {
+
+ final byte[] k1 = encodeBigDecimal(i1);
+
+ final byte[] k2 = encodeBigDecimal(i2);
+
+ final int ret = BytesUtil.compareBytes(k1, k2);
+
+ if (ret != 0) {
+
+ fail("BigDecimal" + //
+ "\ni1=" + i1 + //
+ "\ni2=" + i2 + //
+// "\ns1=" + Arrays.toString(i1.toByteArray())+//
+// "\ns2=" + Arrays.toString(i2.toByteArray())+//
+// "\nu1=" + BytesUtil.toString(i1.toByteArray())+//
+// "\nu2=" + BytesUtil.toString(i2.toByteArray())+//
+ "\nk1=" + BytesUtil.toString(k1) + //
+ "\nk2=" + BytesUtil.toString(k2) + //
+ "\nret=" + (ret == 0 ? "EQ" : (ret < 0 ? "LT" : "GT"))//
+ );
+
+ }
+
+ }
+
protected void doLTTest(final BigDecimal i1, final BigDecimal i2) {
final byte[] k1 = encodeBigDecimal(i1);
@@ -1748,6 +1774,43 @@
}
+ public void test_BigDecimal_zeros() {
+
+ final BigDecimal z1 = new BigDecimal("0.0");
+ final BigDecimal negz1 = new BigDecimal("-0.0");
+ final BigDecimal z2 = new BigDecimal("0.00");
+ final BigDecimal p1 = new BigDecimal("0.01");
+ final BigDecimal negp1 = new BigDecimal("-0.01");
+ final BigDecimal z3 = new BigDecimal("0000.00");
+ final BigDecimal m1 = new BigDecimal("1.5");
+ final BigDecimal m2 = new BigDecimal("-1.51");
+ final BigDecimal m5 = new BigDecimal("5");
+ final BigDecimal m53 = new BigDecimal("5.000");
+ final BigDecimal m500 = new BigDecimal("00500");
+ final BigDecimal m5003 = new BigDecimal("500.000");
+
+ doLTTest(z1, p1);
+ doLTTest(negp1, z1);
+ doLTTest(negp1, p1);
+ doEQTest(z1, negz1);
+
+ doEncodeDecodeTest(m5);
+ doEncodeDecodeTest(z1);
+ doEncodeDecodeTest(z2);
+ doEncodeDecodeTest(z3);
+ doEncodeDecodeTest(m1);
+ doEncodeDecodeTest(m2);
+
+ doEQTest(m5, m53);
+ doEQTest(m500, m5003);
+ doEQTest(z3, z2);
+ doEQTest(z1, z2);
+ doEQTest(z1, z3);
+ doLTTest(z1, m1);
+ doLTTest(m2, z2);
+ doLTTest(z3, m1);
+
+ }
/**
* Unit tests for encoding {@link BigInteger} keys.
*/
@@ -2119,7 +2182,7 @@
* Stress test with random byte[]s from which we then construct
* {@link BigDecimal}s.
*/
- public void test_BigDecimal_stress_byteArray_values() {
+ public void badTest_BigDecimal_stress_byteArray_values() {
final Random r = new Random();
@@ -2408,14 +2471,14 @@
new BigDecimal(".0"), // scale=1, precision=1
// NB: The precision is the #of decimal digits in the unscaled value.
// NB: scaled := unscaled * (10 ^ -scale)
- new BigDecimal(".010"), // scale=3, precision=2
- new BigDecimal(".01"), // scale=2, precision=1
- new BigDecimal(".1"), // scale=1, precision=1
- new BigDecimal("1."), // scale=0, precision=1
- new BigDecimal("10."), // scale=0, precision=2
- new BigDecimal("10.0"), // scale=1, precision=3
- new BigDecimal("010.0"), // scale=1, precision=3
- new BigDecimal("0010.00"), // scale=2, precision=4
+// new BigDecimal(".010"), // scale=3, precision=2
+// new BigDecimal(".01"), // scale=2, precision=1
+// new BigDecimal(".1"), // scale=1, precision=1
+// new BigDecimal("1."), // scale=0, precision=1
+// new BigDecimal("10."), // scale=0, precision=2
+// new BigDecimal("10.0"), // scale=1, precision=3
+// new BigDecimal("010.0"), // scale=1, precision=3
+// new BigDecimal("0010.00"), // scale=2, precision=4
// @todo Test with cases where scale is negative (large powers of 10).
};
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2010-07-26 17:17:58
|
Revision: 3292
http://bigdata.svn.sourceforge.net/bigdata/?rev=3292&view=rev
Author: martyncutcher
Date: 2010-07-26 17:17:51 +0000 (Mon, 26 Jul 2010)
Log Message:
-----------
Fix problem with zero so that scale is MAX_INT such that "0" is LT "0.0000001"
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-26 16:24:33 UTC (rev 3291)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-26 17:17:51 UTC (rev 3292)
@@ -999,30 +999,22 @@
*/
public KeyBuilder append(final BigDecimal d) {
final int sign = d.signum();
- String unscaledStr = d.unscaledValue().toString();
+ BigDecimal nd = d.stripTrailingZeros();
+
+ String unscaledStr = nd.unscaledValue().toString();
+
final int precision;
final int scale;
if ("0".equals(unscaledStr)) {
precision = 1;
- scale = 1;
+ scale = Integer.MAX_VALUE;
} else {
- int trailingZeros = 0;
- int strLen = unscaledStr.length();
- for (int i = strLen-1; i > 0 && unscaledStr.charAt(i) == '0'; i--) {
- trailingZeros++;
- }
- if (trailingZeros > 0) {
- unscaledStr = unscaledStr.substring(0, strLen-trailingZeros);
- }
- precision = d.precision() - trailingZeros;
- scale = d.scale() - trailingZeros;
+ precision = nd.precision();
+ scale = nd.scale();
}
- // Special case for zero
- boolean isZero = "0".equals(unscaledStr);
-
int exponent = precision - scale;
if (sign == -1) {
exponent = -exponent;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2010-07-26 16:24:39
|
Revision: 3291
http://bigdata.svn.sourceforge.net/bigdata/?rev=3291&view=rev
Author: martyncutcher
Date: 2010-07-26 16:24:33 +0000 (Mon, 26 Jul 2010)
Log Message:
-----------
Add BigDecimal normalization to remove trailing zeros as explicit precision. This ensures that abc.00 is equal to abc.0 and abc. A special-case is required for zero values to ensure the same rule - 0 equal 0.00
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-26 16:12:45 UTC (rev 3290)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-26 16:24:33 UTC (rev 3291)
@@ -986,26 +986,53 @@
* {@link #flipDigits(String)} for the equivalent of 2s compliment negative
* representation.
*
- * @todo document the handling of trailing zeros. these are normalized out
- * as they reflect a quality of precision which is not preserved by
- * the corresponding sort key.
+ * There are two cases where scale and trailing zeros interact. The
+ * case of "0.000" is represented as precision of 1 and scale of 3,
+ * indicating the "0" is shifted down 3 decimal places. While "5.000"
+ * is represented as precision of 4 and scale of 3. The special case
+ * of zero is allowed because shifting zero to the right leaves a new
+ * zero on the left, so a zero value must be checked for explicitly, while
+ * if we want to compare "5", "5.00" and "5.0000" as equal we must
+ * remove and compensate for trailing zeros.
*
* @see #decodeBigDecimal(int, byte[])
*/
public KeyBuilder append(final BigDecimal d) {
final int sign = d.signum();
- final int precision = d.precision();
- final int scale = d.scale();
+ String unscaledStr = d.unscaledValue().toString();
+
+ final int precision;
+ final int scale;
+
+ if ("0".equals(unscaledStr)) {
+ precision = 1;
+ scale = 1;
+ } else {
+ int trailingZeros = 0;
+ int strLen = unscaledStr.length();
+ for (int i = strLen-1; i > 0 && unscaledStr.charAt(i) == '0'; i--) {
+ trailingZeros++;
+ }
+ if (trailingZeros > 0) {
+ unscaledStr = unscaledStr.substring(0, strLen-trailingZeros);
+ }
+ precision = d.precision() - trailingZeros;
+ scale = d.scale() - trailingZeros;
+ }
+
+ // Special case for zero
+ boolean isZero = "0".equals(unscaledStr);
+
int exponent = precision - scale;
if (sign == -1) {
exponent = -exponent;
}
append((byte) sign);
- append(exponent);
+ append(exponent);
// Note: coded as digits
- String unscaledStr = d.unscaledValue().toString();
+
if (sign == -1) {
unscaledStr = flipDigits(unscaledStr);
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <btm...@us...> - 2010-07-26 16:12:52
|
Revision: 3290
http://bigdata.svn.sourceforge.net/bigdata/?rev=3290&view=rev
Author: btmurphy
Date: 2010-07-26 16:12:45 +0000 (Mon, 26 Jul 2010)
Log Message:
-----------
[trunk]: trac #115 - Provide a means to setup and tear down a federation running on a single server to support operator directed benchmarking and CI performance tests. [Additional changes to method NicUtil.getIpAddress(systemPropertyName, defaultNic, loopbackOk) and related config files to make it easier to use that method in both deployment and test scrnarios
Modified Paths:
--------------
trunk/bigdata/src/java/com/bigdata/util/config/NicUtil.java
trunk/bigdata-jini/src/java/com/bigdata/service/jini/util/LookupStarter.java
trunk/bigdata-jini/src/java/com/bigdata/service/jini/util/config/lookup.config
trunk/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
trunk/bigdata-jini/src/test/com/bigdata/jini/start/testReggie.config
trunk/bigdata-jini/src/test/com/bigdata/jini/start/testStartJini.config
trunk/bigdata-jini/src/test/com/bigdata/zookeeper/testzoo.config
trunk/src/resources/config/jini/reggie.config
Modified: trunk/bigdata/src/java/com/bigdata/util/config/NicUtil.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/util/config/NicUtil.java 2010-07-26 15:13:46 UTC (rev 3289)
+++ trunk/bigdata/src/java/com/bigdata/util/config/NicUtil.java 2010-07-26 16:12:45 UTC (rev 3290)
@@ -507,54 +507,42 @@
/**
* Special-purpose convenience method that returns a
* <code>String</code> value representing the ip address of
- * the current node; where the value that is returned is
- * determined according to following criteria:
+ * the current node.
* <p>
- * <ul>
- * <li> If a non-<code>null</code> value is input for the
- * <code>systemPropertyName</code> parameter, then
- * this is viewed as a declaration by the caller that
- * that the system property with that given value
- * should take precedence over all other means of
- * determining the desired ip address. As such, this
- * method determines if a system property having the
- * given has indeed been set and, if it has, returns
- * the ip address of the nic having that name; or
- * <code>null</code> if there is no nic with the
- * desired name installed on the node.
- * <li> If a non-<code>null</code> value is input for the
- * <code>systemPropertyName</code> parameter, but
- * no system property with that name has been set,
- * and <code>true</code> has been passed in for
- * the <code>fallbackOk</code> parameter, then this
- * method will return the IPV4 based address of the
- * first reachable nic that is found on the node.
- * Upon failing to find such an address, if the
- * <code>loopbackOk</code> parameter is also
- * <code>true</code>, then this method will return
- * the <i>loop back</i> address of the node; otherwise
- * <code>null</code> is returned.
- * <li> If <code>null</code> is input for the
- * <code>systemPropertyName</code> parameter, but
- * a non-<code>null</code> value is input for the
- * <code>defaultNic</code> parameter, then this
- * method returns the ip address of the nic having
- * that name; or <code>null</code> if there is no
- * nic with the desired default name installed on the
- * node.
- * <li> If <code>null</code> is input for both the
- * <code>systemPropertyName</code> parameter and the
- * <code>defaultNic</code> parameter, and if the
- * <code>fallbackOk</code> parameter is <code>true</code>,
- * then this method will return the IPV4 based address
- * of the first reachable nic that is found on the node.
- * Upon failing to find such an address, if the
- * <code>loopbackOk</code> parameter is also
- * <code>true</code>, then this method will return
- * the <i>loop back</i> address of the node; otherwise
- * <code>null</code> is returned.
- * </ul>
+ * If a non-<code>null</code> value is input for the
+ * <code>systemPropertyName</code> parameter, then this
+ * method first determines if a system property with
+ * name equivalent to the given value has been set and,
+ * if it has, returns the ip address of the nic whose name
+ * is equivalent to that system property value; or
+ * <code>null</code> if there is no nic with the desired
+ * name installed on the node.
* <p>
+ * If there is no system property whose name is the value
+ * of the <code>systemPropertyName</code> parameter, and
+ * if the value "default" is input for the
+ * <code>defaultNic</code> parameter, then this method
+ * will return the IPV4 based address of the first reachable
+ * nic that can be found on the node; otherwise, if a
+ * non-<code>null</code> value not equal to "default" is
+ * input for the the <code>defaultNic</code> parameter,
+ * then this method returns the ip address of the nic
+ * corresponding to that given name; or <code>null</code>
+ * if there is no such nic name installed on the node.
+ * <p>
+ * If, on the other hand, <code>null</code> is input for
+ * the <code>systemPropertyName</code> parameter, then
+ * this method will attempt to find the desired ip address
+ * using only the value of the <code>defaultNic</code>,
+ * and applying the same search criteria as described
+ * above.
+ * <p>
+ * Note that in all cases, if <code>true</code> is input
+ * for the <code>loopOk</code> parameter, then upon failing
+ * to find a valid ip address using the specified search
+ * mechanism, this method will return the <i>loop back</i>
+ * address; otherwise, <code>null</code> is returned.
+ * <p>
* This method can be called from within a configuration
* as well as from within program control.
*
@@ -568,33 +556,24 @@
* the name of the network interface
* whose ip address should be returned
* if <code>null</code> is input for the
+ * <code>systemPropertyName</code> parameter,
+ * or if there is no system property with
+ * name equivalent the value of the
* <code>systemPropertyName</code> parameter.
*
- * @param fallbackOk if <code>true</code>, then if either
- * no system property is set having the
- * name referenced by the
- * <code>systemPropertyName</code> parameter,
- * or if <code>null</code> is input for both
- * the <code>systemPropertyName</code>
- * parameter and the <code>defaultNic</code>
- * parameter, return the IPV4 based address
- * of the first reachable network interface
- * that can be found on the node.
+ * @param loopbackOk if <code>true</code>, then return the
+ * <i>loop back</i> address upon failure
+ * to find a valid ip address using the
+ * search criteria specified through the
+ * <code>systemPropertyName</code> and
+ * <code>defaultNic</code> parameters.
*
- * @param loopbackOk if <code>true</code>, and if <code>true</code>
- * is also input for the <code>fallbackOk</code>
- * parameter, then if this method attempts,
- * but fails, to find a valid IPV4 fallback
- * address, then the node's <i>loop back</i>
- * address is returned.
- *
* @return a <code>String</code> representing an ip address associated
* with the current node; where the value that is returned is
* determined according to the criteria described above.
*/
public static String getIpAddress(String systemPropertyName,
String defaultNic,
- boolean fallbackOk,
boolean loopbackOk)
throws SocketException, IOException
{
@@ -613,32 +592,30 @@
}
if(propSet) {
return getIpAddress(nicName, 0, loopbackOk);
- } else {//desired system property not set, try fallback
- if(fallbackOk) {
- return getDefaultIpv4Address(loopbackOk);
+ } else {//system property not set, try default and/or fallback
+ if(defaultNic != null) {
+ if( defaultNic.equals("default") ) {
+ return getDefaultIpv4Address(loopbackOk);
+ } else {
+ return getIpAddress(defaultNic, 0, loopbackOk);
+ }
} else {
return null;
}
}
- } else {//no system property name provided, try default nic
+ } else {//no system property name provided, try default
if(defaultNic != null) {
- return getIpAddress(defaultNic, 0, loopbackOk);
- } else {//no default nic provided, try fallback
- if(fallbackOk) {
+ if( defaultNic.equals("default") ) {
return getDefaultIpv4Address(loopbackOk);
} else {
- return null;
+ return getIpAddress(defaultNic, 0, loopbackOk);
}
+ } else {
+ return getIpAddress(null, loopbackOk);
}
}
}
- public static String getIpAddress()
- throws SocketException, IOException
- {
- return getIpAddress(null, null, true, true);
- }
-
/**
* Examines each address associated with each network interface
* card (nic) installed on the current node, and returns the
Modified: trunk/bigdata-jini/src/java/com/bigdata/service/jini/util/LookupStarter.java
===================================================================
--- trunk/bigdata-jini/src/java/com/bigdata/service/jini/util/LookupStarter.java 2010-07-26 15:13:46 UTC (rev 3289)
+++ trunk/bigdata-jini/src/java/com/bigdata/service/jini/util/LookupStarter.java 2010-07-26 16:12:45 UTC (rev 3290)
@@ -74,8 +74,7 @@
private static String defaultGroup = null;
static {
try {
- thisHost = NicUtil.getIpAddress
- ("default.nic", null, true, true);
+ thisHost = NicUtil.getIpAddress("default.nic", "default", true);
defaultGroup =
System.getProperty("bigdata.fedname",
"bigdata.test.group-"+thisHost);
Modified: trunk/bigdata-jini/src/java/com/bigdata/service/jini/util/config/lookup.config
===================================================================
--- trunk/bigdata-jini/src/java/com/bigdata/service/jini/util/config/lookup.config 2010-07-26 15:13:46 UTC (rev 3289)
+++ trunk/bigdata-jini/src/java/com/bigdata/service/jini/util/config/lookup.config 2010-07-26 16:12:45 UTC (rev 3290)
@@ -10,7 +10,7 @@
com.sun.jini.reggie {
private static exportPort = 0;//randomly chosen port
private static exportIpAddr =
- NicUtil.getIpAddress("default.nic", null, true, true);
+ NicUtil.getIpAddress("default.nic", "default", true);
private static codebasePort = 23333;//value used if not set by system prop
// Public configuration entries
Modified: trunk/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
===================================================================
--- trunk/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config 2010-07-26 15:13:46 UTC (rev 3289)
+++ trunk/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config 2010-07-26 16:12:45 UTC (rev 3290)
@@ -77,7 +77,7 @@
static serviceDir = ConfigMath.getAbsoluteFile(new File(fedname));
private static localIpAddr =
- NicUtil.getIpAddress("default.nic", null, true, true);
+ NicUtil.getIpAddress("default.nic", "default", true);
/*
* Example cluster configuration.
Modified: trunk/bigdata-jini/src/test/com/bigdata/jini/start/testReggie.config
===================================================================
--- trunk/bigdata-jini/src/test/com/bigdata/jini/start/testReggie.config 2010-07-26 15:13:46 UTC (rev 3289)
+++ trunk/bigdata-jini/src/test/com/bigdata/jini/start/testReggie.config 2010-07-26 16:12:45 UTC (rev 3290)
@@ -15,7 +15,7 @@
com.sun.jini.reggie {
private static exportIpAddr =
- NicUtil.getIpAddress("default.nic", null, true, true);
+ NicUtil.getIpAddress("default.nic", "default", true);
private static exportPort = Integer.parseInt("0");
private static serverILFactory =
Modified: trunk/bigdata-jini/src/test/com/bigdata/jini/start/testStartJini.config
===================================================================
--- trunk/bigdata-jini/src/test/com/bigdata/jini/start/testStartJini.config 2010-07-26 15:13:46 UTC (rev 3289)
+++ trunk/bigdata-jini/src/test/com/bigdata/jini/start/testStartJini.config 2010-07-26 16:12:45 UTC (rev 3290)
@@ -15,7 +15,7 @@
private static appHome = System.getProperty("app.home", ConfigMath.getAbsolutePath(new File(".")) );
private static codebaseHost =
- NicUtil.getIpAddress("default.nic", null, true, true);
+ NicUtil.getIpAddress("default.nic", "default", true);
private static codebasePort = "23334";
private static defaultCodebaseRootDir =
ConfigUtil.concat( new String[] { appHome, "${/}bigdata-jini${/}lib${/}jini${/}lib-dl" } );
Modified: trunk/bigdata-jini/src/test/com/bigdata/zookeeper/testzoo.config
===================================================================
--- trunk/bigdata-jini/src/test/com/bigdata/zookeeper/testzoo.config 2010-07-26 15:13:46 UTC (rev 3289)
+++ trunk/bigdata-jini/src/test/com/bigdata/zookeeper/testzoo.config 2010-07-26 16:12:45 UTC (rev 3290)
@@ -34,7 +34,7 @@
private static appHome =
System.getProperty("app.home", ConfigMath.getAbsolutePath(new File(".")) );
private static localIpAddr =
- NicUtil.getIpAddress("default.nic", null, true, true);
+ NicUtil.getIpAddress("default.nic", "default", true);
/* A comma delimited list of the known zookeeper servers together
* with their assigned "myid".
Modified: trunk/src/resources/config/jini/reggie.config
===================================================================
--- trunk/src/resources/config/jini/reggie.config 2010-07-26 15:13:46 UTC (rev 3289)
+++ trunk/src/resources/config/jini/reggie.config 2010-07-26 16:12:45 UTC (rev 3290)
@@ -14,7 +14,8 @@
com.sun.jini.reggie {
- private static exportIpAddr = NicUtil.getIpAddress(null, null, true, true);
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", "default", false);
private static exportPort =
Integer.parseInt( System.getProperty("exportPort", "0") );
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2010-07-26 15:13:53
|
Revision: 3289
http://bigdata.svn.sourceforge.net/bigdata/?rev=3289&view=rev
Author: mrpersonick
Date: 2010-07-26 15:13:46 +0000 (Mon, 26 Jul 2010)
Log Message:
-----------
hiding the NullIV class from the world at large
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/Justification.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractIV.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractInlineIV.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IV.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/NullIV.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/TermId.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/Justification.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/Justification.java 2010-07-26 14:24:18 UTC (rev 3288)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/Justification.java 2010-07-26 15:13:46 UTC (rev 3289)
@@ -37,7 +37,6 @@
import com.bigdata.journal.TemporaryRawStore;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.IVUtility;
-import com.bigdata.rdf.internal.NullIV;
import com.bigdata.rdf.internal.TermId;
import com.bigdata.rdf.model.StatementEnum;
import com.bigdata.rdf.rules.InferenceEngine;
@@ -442,16 +441,8 @@
for (int i = 0; i < ivs.length; i++) {
- if (ivs[i] != null) {
-
- ivs[i].encode(keyBuilder);
-
- } else {
-
- NullIV.INSTANCE.encode(keyBuilder);
-
- }
-
+ IVUtility.encode(keyBuilder, ivs[i]);
+
}
return keyBuilder.getKey();
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractIV.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractIV.java 2010-07-26 14:24:18 UTC (rev 3288)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractIV.java 2010-07-26 15:13:46 UTC (rev 3289)
@@ -603,8 +603,7 @@
switch (dte) {
case XSDInt:
- final int i = ((NumericBNodeIV) this).getInlineValue().intValue();
- keyBuilder.append(i);
+ keyBuilder.append(((Integer) getInlineValue()).intValue());
break;
case UUID:
keyBuilder.append((UUID) getInlineValue());
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractInlineIV.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractInlineIV.java 2010-07-26 14:24:18 UTC (rev 3288)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractInlineIV.java 2010-07-26 15:13:46 UTC (rev 3289)
@@ -83,13 +83,6 @@
}
/**
- * Always returns <code>false</code> since the value is inline.
- */
- final public boolean isNull() {
- return false;
- }
-
- /**
* No term identifier for an inline IV - throws an exception.
*/
final public long getTermId() {
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IV.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IV.java 2010-07-26 14:24:18 UTC (rev 3288)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IV.java 2010-07-26 15:13:46 UTC (rev 3289)
@@ -128,14 +128,6 @@
boolean isTermId();
/**
- * <code>true</code> iff the RDF value is a term identifier whose value is
- * <code>0L</code>.
- *
- * @see #isTermId()
- */
- boolean isNull();
-
- /**
* Return the term identifier.
*
* @return The term identifier.
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java 2010-07-26 14:24:18 UTC (rev 3288)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java 2010-07-26 15:13:46 UTC (rev 3289)
@@ -31,6 +31,7 @@
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.UUID;
+import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.rawstore.Bytes;
import com.bigdata.rdf.internal.constraints.AbstractInlineConstraint;
@@ -162,6 +163,33 @@
}
/**
+ * Encode an RDF value into a key for one of the statement indices. Handles
+ * null {@link IV} references gracefully.
+ *
+ * @param keyBuilder
+ * The key builder.
+ * @param iv
+ * The internal value (can be <code>null</code>).
+ *
+ * @return The key builder.
+ */
+ public static IKeyBuilder encode(final IKeyBuilder keyBuilder, final IV iv) {
+
+ if (iv == null) {
+
+ NullIV.INSTANCE.encode(keyBuilder);
+
+ } else {
+
+ iv.encode(keyBuilder);
+
+ }
+
+ return keyBuilder;
+
+ }
+
+ /**
* Decode an {@link IV} from a byte[].
*
* @param key
@@ -262,13 +290,6 @@
final byte flags = KeyBuilder.decodeByte(key[o++]);
-// /*
-// * FIXME iNull does not work yet
-// */
-// if (AbstractIV.isNull(flags))
-// return null;
-//
-
/*
* Handle a term identifier (versus an inline value).
*/
@@ -277,10 +298,6 @@
// decode the term identifier.
final long termId = KeyBuilder.decodeLong(key, o);
-// /*
-// * FIXME this is here for now until
-// * {@link AbstractInternalValue#isNull(byte)} works.
-// */
if (termId == TermId.NULL)
return null;
else
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/NullIV.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/NullIV.java 2010-07-26 14:24:18 UTC (rev 3288)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/NullIV.java 2010-07-26 15:13:46 UTC (rev 3289)
@@ -28,12 +28,7 @@
package com.bigdata.rdf.internal;
-/**
- * FIXME A null {@link IV} class used in cases where we want to encode and
- * decode null in the key space. We really need to have a null option in the
- * byte flags so we can avoid this class altogether.
- */
-public class NullIV extends TermId {
+class NullIV extends TermId {
private static final long serialVersionUID = -214758033769962923L;
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/TermId.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/TermId.java 2010-07-26 14:24:18 UTC (rev 3288)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/TermId.java 2010-07-26 15:13:46 UTC (rev 3289)
@@ -23,6 +23,8 @@
*/
package com.bigdata.rdf.internal;
+import java.util.UUID;
+import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.rawstore.Bytes;
import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.model.BigdataValueFactory;
@@ -174,13 +176,6 @@
}
/**
- * Return <code>true</code> iff the term identifier is null.
- */
- final public boolean isNull() {
- return termId == NULL;
- }
-
- /**
* Note: only the termId matters for equality (unless we also have a
* transient reference to the value factory which stands in as a proxy for
* the KB instance).
@@ -225,5 +220,22 @@
return termId < termId2 ? -1 : termId > termId2 ? 1 : 0;
}
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public IKeyBuilder encode(final IKeyBuilder keyBuilder) {
+
+ // First emit the flags byte.
+ keyBuilder.append(flags());
+
+ keyBuilder.append(getTermId());
+
+ return keyBuilder;
+
+ }
+
+
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java 2010-07-26 14:24:18 UTC (rev 3288)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java 2010-07-26 15:13:46 UTC (rev 3289)
@@ -46,7 +46,6 @@
import com.bigdata.rdf.internal.DTE;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.IVUtility;
-import com.bigdata.rdf.internal.NullIV;
import com.bigdata.rdf.internal.TermId;
import com.bigdata.rdf.internal.UUIDLiteralIV;
import com.bigdata.rdf.internal.VTE;
@@ -515,7 +514,7 @@
IV iv = spo.get(a[i]);
- encodeKey(keyBuilder, iv);
+ IVUtility.encode(keyBuilder, iv);
}
@@ -524,37 +523,6 @@
}
/**
- * Encode an RDF value into a key for one of the statement indices.
- *
- * @param keyBuilder
- * The key builder.
- * @param iv
- * The RDF value.
- *
- * @return The key builder.
- */
- private void encodeKey(final IKeyBuilder keyBuilder, final IV iv) {
-
- if (iv == null) {
-
- /*
- * FIXME justifications use null IVs to represent wildcards. we
- * use a fake TermId object to simulate this since we have no way
- * of encoding null into the byte flags yet.
- */
- NullIV.INSTANCE.encode(keyBuilder);
-
- return;
-
- } else {
-
- iv.encode(keyBuilder);
-
- }
-
- }
-
- /**
* Decode the key into an {@link SPO}. The {@link StatementEnum} and the
* optional SID will not be decoded, since it is carried in the B+Tree
* value. However, if the {@link SPOKeyOrder} is a quad order then the
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java 2010-07-26 14:24:18 UTC (rev 3288)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java 2010-07-26 15:13:46 UTC (rev 3289)
@@ -164,12 +164,6 @@
assertFalse(v.isInline());
- if (termId == 0L) {
- assertTrue(v.toString(), v.isNull());
- } else {
- assertFalse(v.toString(), v.isNull());
- }
-
assertEquals(termId, v.getTermId());
try {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-07-26 14:24:25
|
Revision: 3288
http://bigdata.svn.sourceforge.net/bigdata/?rev=3288&view=rev
Author: thompsonbry
Date: 2010-07-26 14:24:18 +0000 (Mon, 26 Jul 2010)
Log Message:
-----------
Working through xsd:decimal issues with MartynC.
Moved much of the byteLength() logic for xsd:integer and xsd:decimal onto KeyBuilder.
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDIntegerIV.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-26 02:16:07 UTC (rev 3287)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-26 14:24:18 UTC (rev 3288)
@@ -247,7 +247,7 @@
* {@link #buf buffer} may be grown by this operation but it will not be
* truncated.
* <p>
- * This operation is equivilent to
+ * This operation is equivalent to
*
* <pre>
* ensureCapacity(this.len + len)
@@ -945,28 +945,57 @@
}
- // The encoding ot a BigDecimal requires the expression
- // of scale and length
- // BigDecimal.scale indicates the precision of the number, where
- // '3' is three decimal places and '-3' rounded to '000'
- // BigDecimal.precision() is the number of unscaled digits
- // therefore the precision - scale is an expression of the
- // exponent of the normalised number.
- // This means that the exponent could be zero or negative so the sign of the
- // number cannot be indicated by adding to the exponent.
- // Instead an explicit sign byte,'0' or '1' is used.
- // The actual BigDecimal serialization uses the String conversions
- // supported by BigDecimal, less the '-' sign character if applicable.
- // The length of this data is terminated by a zero byte.
- //
- // The variable encoding of BigNumbers requires this String representation
- // and negative representations are further encoded using flipDigits
- // for the equivalent of 2s compliment negative representation.
-
+ /**
+ * Return the #of bytes in the unsigned byte[] representation of the
+ * {@link BigInteger} value.
+ *
+ * @param value
+ * The {@link BigInteger} value.
+ *
+ * @return The byte length of its unsigned byte[] representation.
+ */
+ static public int byteLength(final BigInteger value) {
+
+ return 2/* runLength */+ (value.bitLength() / 8 + 1)/* data */;
+
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * Note: Precision is NOT preserved by this encoding.
+ * <h2>Implementation details</h2>
+ * The encoding to a BigDecimal requires the expression of scale and length
+ * {@link BigDecimal#scale()} indicates the precision of the number, where
+ * '3' is three decimal places and '-3' rounded to '000'
+ * {@link BigDecimal#precision()} is the number of unscaled digits therefore
+ * <code>precision - scale</code> is an expression of the exponent of the
+ * normalized number. This means that the exponent could be zero or negative
+ * so the sign of the number cannot be indicated by adding to the exponent.
+ * Instead an explicit sign byte,'0' or '1' is used. The actual
+ * {@link BigDecimal} serialization uses the {@link String} conversions
+ * supported by {@link BigDecimal}, less the '-' sign character if
+ * applicable. The length of this data is terminated by a trailing byte. The
+ * value of that byte depends on the sign of the original {@link BigDecimal}
+ * and is used to impose the correct sort order on negative
+ * {@link BigDecimal} values which differ only in the digits in the decimal
+ * portion.
+ *<p>
+ * The variable encoding of BigNumbers requires this String representation
+ * and negative representations are further encoded using
+ * {@link #flipDigits(String)} for the equivalent of 2s compliment negative
+ * representation.
+ *
+ * @todo document the handling of trailing zeros. these are normalized out
+ * as they reflect a quality of precision which is not preserved by
+ * the corresponding sort key.
+ *
+ * @see #decodeBigDecimal(int, byte[])
+ */
public KeyBuilder append(final BigDecimal d) {
- int sign = d.signum();
- int precision = d.precision();
- int scale = d.scale();
+ final int sign = d.signum();
+ final int precision = d.precision();
+ final int scale = d.scale();
int exponent = precision - scale;
if (sign == -1) {
exponent = -exponent;
@@ -975,16 +1004,43 @@
append((byte) sign);
append(exponent);
+ // Note: coded as digits
String unscaledStr = d.unscaledValue().toString();
if (sign == -1) {
unscaledStr = flipDigits(unscaledStr);
}
appendASCII(unscaledStr); // the unscaled BigInteger representation
- append((byte) 0);
+ // Note: uses unsigned 255 if negative and unsigned 0 if positive.
+ append(sign == -1 ? (byte) Byte.MAX_VALUE: (byte) 0);
return this;
}
+ /**
+ * Return the #of bytes in the unsigned byte[] representation of the
+ * {@link BigDecimal} value.
+ *
+ * @param value
+ * The {@link BigDecimal} value.
+ *
+ * @return The byte length of its unsigned byte[] representation.
+ */
+ static public int byteLength(final BigDecimal value) {
+
+ // FIXME Make sure to normalize trailing zeros first if necessary.
+ final int dataLen = value.unscaledValue().toString().length();
+
+ final int byteLength =
+ + 1 /* sign */
+ + 4 /* exponent */
+ + dataLen /* data */
+ + 1 /* termination byte */
+ ;
+
+ return byteLength;
+
+ }
+
/*
* static helper methods.
*/
@@ -1420,47 +1476,73 @@
}
- private static final byte eos = decodeByte(0);
- private static final byte negSign = decodeByte(-1);
-
- // FIXME decodeBigDecimal(int, byte[])
+ /**
+ * Decodes a {@link BigDecimal} key, returning a byte[] which may be used to
+ * construct a {@link BigDecimal} having the decoded value.
+ *
+ * The number of bytes consumed by the key component is
+ * <code>2 + runLength</2>. The
+ * <code>2</code> is a fixed length field coding the signum of the value and
+ * its runLength. The length of the returned array is the runLength of the
+ * variable length portion of the value.
+ *
+ * This method may be used to scan through a key containing
+ * {@link BigDecimal} components.
+ *
+ * @param offset
+ * The offset of the start of the {@link BigDecimal} in the key.
+ * @param key
+ * The key.
+ * @return The byte[] to be passed to {@link BigDecimal#BigInteger(byte[])}.
+ *
+ * @todo update javadoc
+ *
+ * FIXME We need a version which has all the metadata to support scanning
+ * through a key as well as one that does a simple decode.
+ */
static public BigDecimal decodeBigDecimal(final int offset, final byte[] key) {
int curs = offset;
- byte sign = key[curs++];
+ final byte sign = key[curs++];
int exponent = decodeInt(key, curs);
- boolean neg = sign == negSign;
+ final boolean neg = sign == negSign;
if (neg) {
exponent = -exponent;
}
curs += 4;
int len = 0;
- for (int i = curs; key[i] != eos; i++) len++;
+ for (int i = curs; key[i] != (neg ? eos2 : eos); i++) len++;
String unscaledStr = decodeASCII(key, curs, len);
if (neg) {
unscaledStr = flipDigits(unscaledStr);
}
- BigInteger unscaled = new BigInteger(unscaledStr);
+ final BigInteger unscaled = new BigInteger(unscaledStr);
- int precision = len;
- int scale = precision - exponent - (neg ? 1 : 0);
+ final int precision = len;
+ final int scale = precision - exponent - (neg ? 1 : 0);
- BigDecimal ret = new BigDecimal(unscaled, scale);
+ final BigDecimal ret = new BigDecimal(unscaled, scale);
return ret; // relative scale adjustment
}
- static final char[] flipMap = {'0', '1', '2', '3', '4',
+ private static final byte eos = decodeByte(0);
+ private static final byte eos2 = decodeByte(Byte.MAX_VALUE);
+ private static final byte negSign = decodeByte(-1);
+
+ private static final char[] flipMap = {'0', '1', '2', '3', '4',
'5', '6', '7', '8', '9'
};
-
- /*
- * Flip numbers such that 0/9,1/8,2/7,3/6,4/5
+
+ /**
+ * Flip numbers such that <code>0/9,1/8,2/7,3/6,4/5</code> - this is the
+ * equivalent of a two-complement representation for the base 10 character
+ * digits.
*/
- static private String flipDigits(String str) {
- char[] chrs = str.toCharArray();
+ static private String flipDigits(final String str) {
+ final char[] chrs = str.toCharArray();
for (int i = 0; i < chrs.length; i++) {
- int flip = '9' - chrs[i];
+ final int flip = '9' - chrs[i];
if (flip >= 0 && flip < 10) {
chrs[i] = flipMap[flip];
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java 2010-07-26 02:16:07 UTC (rev 3287)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java 2010-07-26 14:24:18 UTC (rev 3288)
@@ -1995,8 +1995,52 @@
}
}
-
+
/**
+ * Test with positive and negative {@link BigInteger}s having a common
+ * prefix with varying digits after the prefix.
+ */
+ public void test_BigInteger_sortOrder() {
+
+ final BigInteger p1 = new BigInteger("15");
+ final BigInteger p2 = new BigInteger("151");
+ final BigInteger m1 = new BigInteger("-15");
+ final BigInteger m2 = new BigInteger("-151");
+
+ doEncodeDecodeTest(p1);
+ doEncodeDecodeTest(p2);
+ doEncodeDecodeTest(m1);
+ doEncodeDecodeTest(m2);
+
+ doLTTest(p1, p2); // 15 LT 151
+ doLTTest(m1, p1); // -15 LT 15
+ doLTTest(m2, m1); // -151 LT -15
+
+ }
+
+ /**
+ * Test with positive and negative {@link BigDecimal}s having varying
+ * digits after the decimals.
+ */
+ public void test_BigDecimal_negativeSortOrder() {
+
+ final BigDecimal p1 = new BigDecimal("1.5");
+ final BigDecimal p2 = new BigDecimal("1.51");
+ final BigDecimal m1 = new BigDecimal("-1.5");
+ final BigDecimal m2 = new BigDecimal("-1.51");
+
+ doEncodeDecodeTest(p1);
+ doEncodeDecodeTest(p2);
+ doEncodeDecodeTest(m1);
+ doEncodeDecodeTest(m2);
+
+ doLTTest(p1, p2); // 1.5 LT 1.51
+ doLTTest(m1, p1); // -1.5 LT 1.5
+ doLTTest(m2, m1); // -1.51 LT -1.5
+
+ }
+
+ /**
* Stress test with random byte[]s from which we then construct
* {@link BigInteger}s.
*/
@@ -2074,12 +2118,8 @@
/**
* Stress test with random byte[]s from which we then construct
* {@link BigDecimal}s.
- *
- * FIXME: At present the comparisons fail proably due to a failure to
- * understand the limits of the BigDecimal representations - so this
- * test has been deactivated by name prefix.
*/
- public void badTest_BigDecimal_stress_byteArray_values() {
+ public void test_BigDecimal_stress_byteArray_values() {
final Random r = new Random();
@@ -2211,48 +2251,48 @@
// return key;
//
// }
-//
-// /**
-// * Normalize the {@link BigDecimal} by setting the scale such that there are
-// * no digits before the decimal point.
-// *
-// * FIXME This fails for "0" and "0.0". The trailing .0 is considered a
-// * significant digit and is not being stripped. We need to also strip
-// * trailing zeros which are significant.
-// *
-// * <pre>
-// * i=0 (scale=0,prec=1) : 0, scale=0, precision=1, unscaled=0, unscaled_byte[]=[0]
-// * i=0.0 (scale=1,prec=1) : 0.0, scale=1, precision=1, unscaled=0, unscaled_byte[]=[0]
-// * </pre>
-// */
-// private BigDecimal normalizeBigDecimal(final BigDecimal i) {
-//
-// return i.stripTrailingZeros();
-//
-// }
-//
-// /**
-// * Dumps out interesting bits of the {@link BigDecimal} state.
-// *
-// * @return The dump.
-// */
-// private String dumpBigDecimal(final BigDecimal i) {
-//
-// final BigInteger unscaled = i.unscaledValue();
-//
-// final String msg = i.toString() + ", scale=" + i.scale()
-// + //
-// ", precision=" + i.precision()
-// + //
-// ", unscaled=" + unscaled
-// + //
-// ", unscaled_byte[]="
-// + BytesUtil.toString(unscaled.toByteArray())//
-// ;
-//
-// return msg;
-//
-// }
+
+ /**
+ * Normalize the {@link BigDecimal} by setting the scale such that there are
+ * no digits before the decimal point.
+ *
+ * FIXME This fails for "0" and "0.0". The trailing .0 is considered a
+ * significant digit and is not being stripped. We need to also strip
+ * trailing zeros which are significant.
+ *
+ * <pre>
+ * i=0 (scale=0,prec=1) : 0, scale=0, precision=1, unscaled=0, unscaled_byte[]=[0]
+ * i=0.0 (scale=1,prec=1) : 0.0, scale=1, precision=1, unscaled=0, unscaled_byte[]=[0]
+ * </pre>
+ */
+ private BigDecimal normalizeBigDecimal(final BigDecimal i) {
+
+ return i.stripTrailingZeros();
+
+ }
+
+ /**
+ * Dumps out interesting bits of the {@link BigDecimal} state.
+ *
+ * @return The dump.
+ */
+ private String dumpBigDecimal(final BigDecimal i) {
+
+ final BigInteger unscaled = i.unscaledValue();
+
+ final String msg = i.toString() + ", scale=" + i.scale()
+ + //
+ ", precision=" + i.precision()
+ + //
+ ", unscaled=" + unscaled
+ + //
+ ", unscaled_byte[]="
+ + BytesUtil.toString(unscaled.toByteArray())//
+ ;
+
+ return msg;
+
+ }
//
// /**
// * Note: must have normalized representation of the BigDecimal to do
@@ -2308,101 +2348,101 @@
//
// }
//
-// private enum CompareEnum {
-//
-// LT(-1), EQ(0), GT(1);
-//
-// private CompareEnum(final int ret) {
-// this.ret = ret;
-// }
-//
-// private int ret;
-//
-// static public CompareEnum valueOf(final int ret) {
-// if(ret<0) return LT;
-// if(ret>0) return GT;
-// return EQ;
-// }
-//
-// }
-//
-// protected void doCompareTest(BigDecimal i1, BigDecimal i2, final CompareEnum cmp) {
-//
-// i1 = normalizeBigDecimal(i1);
-// i2 = normalizeBigDecimal(i2);
-//
-// final byte[] k1 = encodeBigDecimal(i1);
-//
-// final byte[] k2 = encodeBigDecimal(i2);
-//
-// final int ret = BytesUtil.compareBytes(k1, k2);
-//
-// final CompareEnum cmp2 = CompareEnum.valueOf(ret);
-//
-// if (cmp2 != cmp) {
-//
-// fail("BigDecimal" + //
-// "\ni1=" + dumpBigDecimal(i1) + //
-// "\ni2=" + dumpBigDecimal(i2) + //
-// "\nk1=" + BytesUtil.toString(k1) + //
-// "\nk2=" + BytesUtil.toString(k2) + //
-// "\nret=" + cmp2 +", but expected="+cmp//
-// );
-//
-// }
-//
-// }
-//
-// /**
-// * Test encode/decode for various values of zero.
-// */
-// public void test_BigDecimal0() {
-//
-// final BigDecimal[] a = new BigDecimal[] {
-// new BigDecimal("0"), // scale=0, precision=1
-// new BigDecimal("0."), // scale=0, precision=1
-// new BigDecimal("0.0"), // scale=1, precision=1
-// new BigDecimal("0.00"), // scale=2, precision=1
-// new BigDecimal("00.0"), // scale=1, precision=1
-// new BigDecimal("00.00"),// scale=2, precision=1
-// new BigDecimal(".0"), // scale=1, precision=1
-// // NB: The precision is the #of decimal digits in the unscaled value.
-// // NB: scaled := unscaled * (10 ^ -scale)
-// new BigDecimal(".010"), // scale=3, precision=2
-// new BigDecimal(".01"), // scale=2, precision=1
-// new BigDecimal(".1"), // scale=1, precision=1
-// new BigDecimal("1."), // scale=0, precision=1
-// new BigDecimal("10."), // scale=0, precision=2
-// new BigDecimal("10.0"), // scale=1, precision=3
-// new BigDecimal("010.0"), // scale=1, precision=3
-// new BigDecimal("0010.00"), // scale=2, precision=4
-// // @todo Test with cases where scale is negative (large powers of 10).
-// };
-//
-// for (BigDecimal i : a) {
-// i = i.stripTrailingZeros();
-// System.err.println("i="
-// + i
-// + "\t(scale="
-// + i.scale()
-// + ",prec="
-// + i.precision()
-// + ") : "
-// + dumpBigDecimal(i)
-//// i.scaleByPowerOfTen(i.scale()- i.precision()))
-// );
-// }
-//
-// for (BigDecimal i : a) {
-// doEncodeDecodeTest(i);
-// }
-//
-// for (int i = 0; i < a.length; i++) {
-// for (int j = 0; j < a.length; j++) {
-// doCompareTest(a[i], a[j], CompareEnum.EQ);
-// }
-// }
-//
-// }
+ private enum CompareEnum {
+
+ LT(-1), EQ(0), GT(1);
+
+ private CompareEnum(final int ret) {
+ this.ret = ret;
+ }
+
+ private int ret;
+
+ static public CompareEnum valueOf(final int ret) {
+ if(ret<0) return LT;
+ if(ret>0) return GT;
+ return EQ;
+ }
+
+ }
+
+ protected void doCompareTest(BigDecimal i1, BigDecimal i2, final CompareEnum cmp) {
+
+ i1 = normalizeBigDecimal(i1);
+ i2 = normalizeBigDecimal(i2);
+
+ final byte[] k1 = encodeBigDecimal(i1);
+
+ final byte[] k2 = encodeBigDecimal(i2);
+
+ final int ret = BytesUtil.compareBytes(k1, k2);
+
+ final CompareEnum cmp2 = CompareEnum.valueOf(ret);
+
+ if (cmp2 != cmp) {
+
+ fail("BigDecimal" + //
+ "\ni1=" + dumpBigDecimal(i1) + //
+ "\ni2=" + dumpBigDecimal(i2) + //
+ "\nk1=" + BytesUtil.toString(k1) + //
+ "\nk2=" + BytesUtil.toString(k2) + //
+ "\nret=" + cmp2 +", but expected="+cmp//
+ );
+
+ }
+
+ }
+
+ /**
+ * Test encode/decode for various values of zero.
+ */
+ public void test_BigDecimal0() {
+
+ final BigDecimal[] a = new BigDecimal[] {
+ new BigDecimal("0"), // scale=0, precision=1
+ new BigDecimal("0."), // scale=0, precision=1
+ new BigDecimal("0.0"), // scale=1, precision=1
+ new BigDecimal("0.00"), // scale=2, precision=1
+ new BigDecimal("00.0"), // scale=1, precision=1
+ new BigDecimal("00.00"),// scale=2, precision=1
+ new BigDecimal(".0"), // scale=1, precision=1
+ // NB: The precision is the #of decimal digits in the unscaled value.
+ // NB: scaled := unscaled * (10 ^ -scale)
+ new BigDecimal(".010"), // scale=3, precision=2
+ new BigDecimal(".01"), // scale=2, precision=1
+ new BigDecimal(".1"), // scale=1, precision=1
+ new BigDecimal("1."), // scale=0, precision=1
+ new BigDecimal("10."), // scale=0, precision=2
+ new BigDecimal("10.0"), // scale=1, precision=3
+ new BigDecimal("010.0"), // scale=1, precision=3
+ new BigDecimal("0010.00"), // scale=2, precision=4
+ // @todo Test with cases where scale is negative (large powers of 10).
+ };
+
+ for (BigDecimal i : a) {
+ i = i.stripTrailingZeros();
+ System.err.println("i="
+ + i
+ + "\t(scale="
+ + i.scale()
+ + ",prec="
+ + i.precision()
+ + ") : "
+ + dumpBigDecimal(i)
+// i.scaleByPowerOfTen(i.scale()- i.precision()))
+ );
+ }
+
+ for (BigDecimal i : a) {
+ doEncodeDecodeTest(i);
+ }
+
+ for (int i = 0; i < a.length; i++) {
+ for (int j = 0; j < a.length; j++) {
+ doCompareTest(a[i], a[j], CompareEnum.EQ);
+ }
+ }
+
+ }
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java 2010-07-26 02:16:07 UTC (rev 3287)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java 2010-07-26 14:24:18 UTC (rev 3288)
@@ -27,7 +27,7 @@
import java.math.BigDecimal;
import java.math.BigInteger;
-import com.bigdata.rawstore.Bytes;
+import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.rdf.model.BigdataLiteral;
import com.bigdata.rdf.model.BigdataValueFactory;
@@ -137,23 +137,18 @@
}
public int byteLength() {
+
if (byteLength == 0) {
/*
* Cache the byteLength if not yet set.
*/
- int dataLen = value.unscaledValue().toString().length();
-
- byteLength =
- 1 /* flags */
- + 1 /* sign */
- + 4 /* exponent */
- + dataLen
- + 1 /* data and null termination */;
+ byteLength = 1 /* flags */ + KeyBuilder.byteLength(value);
}
return byteLength;
+
}
@Override
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDIntegerIV.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDIntegerIV.java 2010-07-26 02:16:07 UTC (rev 3287)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDIntegerIV.java 2010-07-26 14:24:18 UTC (rev 3288)
@@ -27,6 +27,7 @@
import java.math.BigDecimal;
import java.math.BigInteger;
+import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.rdf.model.BigdataLiteral;
import com.bigdata.rdf.model.BigdataValueFactory;
@@ -143,7 +144,7 @@
* Cache the byteLength if not yet set.
*/
- byteLength = 1 /* prefix */+ 2/* runLength */+ (value.bitLength() / 8 + 1)/* data */;
+ byteLength = 1 /* prefix */+ KeyBuilder.byteLength(value);
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2010-07-26 02:16:13
|
Revision: 3287
http://bigdata.svn.sourceforge.net/bigdata/?rev=3287&view=rev
Author: mrpersonick
Date: 2010-07-26 02:16:07 +0000 (Mon, 26 Jul 2010)
Log Message:
-----------
fixing some issues for bryan pre-merge to trunk
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNamedGraphs.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSetBinding.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2010-07-25 21:55:39 UTC (rev 3286)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2010-07-26 02:16:07 UTC (rev 3287)
@@ -3238,7 +3238,7 @@
log.debug("value: " + val + " : " + val2 + " ("
+ val2.getIV() + ")");
- if (val2.getIV().isNull()) {
+ if (val2.getIV() == null) {
/*
* Since the term identifier is NULL this value is
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNamedGraphs.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNamedGraphs.java 2010-07-25 21:55:39 UTC (rev 3286)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNamedGraphs.java 2010-07-26 02:16:07 UTC (rev 3287)
@@ -1668,118 +1668,5 @@
}
- /*
- * FIXME MRP Please resolve edit conflict between testSearchQuery() above
- * and this version of that method.
- */
- public void testSearchQuery_version2() throws Exception {
-
- final BigdataSail sail = getSail();
- sail.initialize();
- final BigdataSailRepository repo = new BigdataSailRepository(sail);
- final BigdataSailRepositoryConnection cxn = (BigdataSailRepositoryConnection) repo
- .getConnection();
- cxn.setAutoCommit(false);
- try {
-
- if (!sail.getDatabase().isQuads()) {
-
- log.warn("test requires quads.");
-
- return;
-
- }
-
- final ValueFactory vf = cxn.getValueFactory();
-
- //create context
- final Resource context1 = vf.createURI( "http://example.org" );
-
- //add statement to context1
- cxn.add( vf.createStatement(
- vf.createURI("http://example.org#Chris"),
- RDFS.LABEL,
- vf.createLiteral("Chris") ),
- context1);
- cxn.commit();
-
- //add statement to default graph
- cxn.add( vf.createStatement(
- vf.createURI("http://example.org#Christian"),
- RDFS.LABEL,
- vf.createLiteral("Christian") ) );
- cxn.commit();
-
- {
- //when running this query, we correctly get bindings for both triples
- final String query =
- "select ?x ?y " +
- "where { " +
- " ?y <"+ BD.SEARCH+"> \"Chris\" . " +
- " ?x <"+ RDFS.LABEL.stringValue() + "> ?y . " +
- "}";
-
- final TupleQuery tupleQuery = cxn.prepareTupleQuery(
- QueryLanguage.SPARQL, query);
- final TupleQueryResult result = tupleQuery.evaluate();
-
-// int i = 1;
-// while (result.hasNext()) {
-// System.err.println(i++ + "#: " + result.next());
-// }
-
- final Collection<BindingSet> answer = new LinkedList<BindingSet>();
- answer.add(createBindingSet(//
- new BindingImpl("x", vf.createURI("http://example.org#Christian")),//
- new BindingImpl("y", vf.createLiteral("Christian"))//
- ));
- answer.add(createBindingSet(//
- new BindingImpl("x", vf.createURI("http://example.org#Chris")),//
- new BindingImpl("y", vf.createLiteral("Chris"))//
- ));
-
- compare(result, answer);
-
- }
-
- {
- //however, when running this query, we incorrectly get both results as it should only return bindings for the triple added to context 1.
- String query =
- "select ?x ?y " +
- "where { " +
- " graph <http://example.org> { " +
- " ?y <"+ BD.SEARCH+"> \"Chris\" . " +
- " ?x <"+ RDFS.LABEL.stringValue() + "> ?y ." +
- " } . " +
- "}";
-
- final TupleQuery tupleQuery = cxn.prepareTupleQuery(
- QueryLanguage.SPARQL, query);
- final TupleQueryResult result = tupleQuery.evaluate();
-
-// int i = 1;
-// while (result.hasNext()) {
-// System.err.println(i++ + "#: " + result.next());
-// }
-
- final Collection<BindingSet> answer = new LinkedList<BindingSet>();
- answer.add(createBindingSet(//
- new BindingImpl("x", vf.createURI("http://example.org#Chris")),//
- new BindingImpl("y", vf.createLiteral("Chris"))//
- ));
-
- compare(result, answer);
-
- }
-
- } finally {
-
- cxn.close();
- sail.__tearDownUnitTest();
-
- }
-
- }
-
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSetBinding.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSetBinding.java 2010-07-25 21:55:39 UTC (rev 3286)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSetBinding.java 2010-07-26 02:16:07 UTC (rev 3287)
@@ -45,6 +45,7 @@
import org.openrdf.repository.RepositoryResult;
import org.openrdf.rio.RDFFormat;
import com.bigdata.rdf.axioms.NoAxioms;
+import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.model.BigdataLiteral;
import com.bigdata.rdf.vocab.NoVocabulary;
@@ -139,6 +140,9 @@
Literal snowball = vf.createLiteral("Snowball");
if (repo instanceof BigdataSailRepository) {
// FIXME MRP to reveiew.
+ final IV iv = repo.getDatabase().getIV(snowball);
+ ((BigdataLiteral) snowball).setIV(iv);
+ System.err.println(((BigdataLiteral) snowball).getIV());
// long tid = repo.getDatabase().getTermId(snowball);
// ((BigdataLiteral) snowball).setTermId(tid);
// System.err.println(((BigdataLiteral) snowball).getTermId());
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-07-25 21:55:49
|
Revision: 3286
http://bigdata.svn.sourceforge.net/bigdata/?rev=3286&view=rev
Author: thompsonbry
Date: 2010-07-25 21:55:39 +0000 (Sun, 25 Jul 2010)
Log Message:
-----------
Branch starts at 3285 on the trunk. This is a branch from the trunk immediately prior to merging in the physical schema change required to support inlining of XSD numeric and other datatypes into the statement indices. This branch exists for maintenance and migration purposes only.
Added Paths:
-----------
branches/NO_INLINING_BRANCH/
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-07-25 21:33:29
|
Revision: 3285
http://bigdata.svn.sourceforge.net/bigdata/?rev=3285&view=rev
Author: thompsonbry
Date: 2010-07-25 21:33:21 +0000 (Sun, 25 Jul 2010)
Log Message:
-----------
Merge trunk to branch (Merging r3113 through r3284).
The following conflicts are to be reviewed by MikeP:
TestNamedGraphs. For this file, I've kept both versions of testSearchQuery(), naming the second one as testSearchQuery_version2(). I think that this was linked to something Christian was asking you about. If there is anything that is supposed to be retained by the merge, please edit accordingly and then get rid of testSearchQuery_version2().
TestBindingSets. For this file, there is a problem with an incoming edit for testSetBinding() which presumes tids rather than IVs. Please review the three lines which are commented out starting at line 142.
Please see line 3241 of BigdataSail. I believe that I made the appropriate change to capture your edit in the trunk concerning NULL termIds.
The following conflicts are to be reviewed by MartynC:
Please review the resolution of a conflict in KeyBuilder beginning at 1423. I would also like to talk over the BigDecimal issue tomorrow. (Also, can you please file an issue specifically for this.)
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/Schema.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/util/config/NicUtil.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestICUUnicodeKeyBuilder.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestJDKUnicodeKeyBuilder.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/cache/TestRingBuffer.java
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/sparse/TestKeyEncodeDecode.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/jini/start/MonitorCreatePhysicalServiceLocksTask.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/jini/start/config/AbstractHostConstraint.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerEntry.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/util/LookupStarter.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/util/config/lookup.config
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/config/TestZookeeperServerEntry.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/testReggie.config
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/testStartJini.config
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/test/com/bigdata/zookeeper/testzoo.config
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/TruthMaintenance.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractIV.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNamedGraphs.java
branches/LEXICON_REFACTOR_BRANCH/build.properties
branches/LEXICON_REFACTOR_BRANCH/build.xml
branches/LEXICON_REFACTOR_BRANCH/src/resources/config/bigdataCluster.config
branches/LEXICON_REFACTOR_BRANCH/src/resources/config/bigdataCluster16.config
branches/LEXICON_REFACTOR_BRANCH/src/resources/config/jini/reggie.config
branches/LEXICON_REFACTOR_BRANCH/src/resources/config/jini/startAll.config
branches/LEXICON_REFACTOR_BRANCH/src/resources/config/logging.properties
branches/LEXICON_REFACTOR_BRANCH/src/resources/config/standalone/bigdataStandalone.config
Added Paths:
-----------
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/attr/
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/attr/ServiceInfo.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/disco/
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/disco/DiscoveryTool.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/disco/config/
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/disco/config/disco.config
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/disco/config/logging.properties
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/util/Util.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/util/config/
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSetBinding.java
branches/LEXICON_REFACTOR_BRANCH/src/resources/bin/disco-tool
Removed Paths:
-------------
branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/AbstractUnicodeKeyBuilderTestCase.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/attr/ServiceInfo.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/disco/DiscoveryTool.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/disco/config/
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/disco/config/disco.config
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/disco/config/logging.properties
branches/LEXICON_REFACTOR_BRANCH/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java
Property Changed:
----------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/
branches/LEXICON_REFACTOR_BRANCH/dsi-utils/
branches/LEXICON_REFACTOR_BRANCH/src/resources/bin/config/
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java 2010-07-25 20:26:13 UTC (rev 3284)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java 2010-07-25 21:33:21 UTC (rev 3285)
@@ -10,7 +10,8 @@
/**
* The JDK bundles support for generating Unicode sort keys, but that
- * support does NOT include compressed sort keys.
+ * support does NOT include compressed sort keys and embeds <code>nul</code>
+ * bytes into its Unicode sort keys.
*/
JDK,
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-07-25 20:26:13 UTC (rev 3284)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-07-25 21:33:21 UTC (rev 3285)
@@ -144,6 +144,7 @@
/**
* Representation includes all aspects of the {@link Serializable} state.
*/
+ @Override
public String toString() {
StringBuilder sb = new StringBuilder(getClass().getName());
@@ -270,14 +271,10 @@
/*
* Figure out which collator to use.
- *
- * Note: The default depends on whether or not the ICU library is on
- * the class path. When it is, we always default to the ICU library.
*/
collator = CollatorEnum.valueOf(getProperty(properties,
- Options.COLLATOR, (icu_avail ? CollatorEnum.ICU.toString()
- : CollatorEnum.JDK.toString())));
+ Options.COLLATOR, CollatorEnum.ICU.toString()));
// true iff the collator was _explicitly_ specified.
final boolean explicitCollatorChoice = getProperty(properties,
@@ -349,7 +346,7 @@
* Figure out the collator strength.
*/
- Object strength = null;
+ Object tmpStrength = null;
final String val = getProperty(properties, Options.STRENGTH);
@@ -357,24 +354,24 @@
try {
- strength = StrengthEnum.valueOf(val);
+ tmpStrength = StrengthEnum.valueOf(val);
} catch (RuntimeException ex) {
- strength = Integer.parseInt(val);
+ tmpStrength = Integer.parseInt(val);
}
}
if (log.isInfoEnabled())
- log.info(Options.STRENGTH + "=" + strength);
+ log.info(Options.STRENGTH + "=" + tmpStrength);
/*
* Note: MAY be null (when null, does not override the collator's
* default).
*/
- this.strength = strength;
+ this.strength = tmpStrength;
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-07-25 20:26:13 UTC (rev 3284)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-07-25 21:33:21 UTC (rev 3285)
@@ -28,7 +28,6 @@
import java.util.Locale;
-import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import com.ibm.icu.text.Collator;
@@ -90,18 +89,6 @@
protected static final Logger log = Logger.getLogger(ICUSortKeyGenerator.class);
/**
- * True iff the {@link #log} level is INFO or less.
- */
- final protected boolean INFO = log.getEffectiveLevel().toInt() <= Level.INFO
- .toInt();
-
- /**
- * True iff the {@link #log} level is DEBUG or less.
- */
- final protected boolean DEBUG = log.getEffectiveLevel().toInt() <= Level.DEBUG
- .toInt();
-
- /**
* Used to encode unicode strings into compact byte[]s that have the same
* sort order (aka sort keys).
*/
@@ -128,7 +115,7 @@
this.locale = locale;
- if(INFO) log.info("locale="+locale);
+ if(log.isInfoEnabled()) log.info("locale="+locale);
this.collator = (RuleBasedCollator) Collator.getInstance(locale);
@@ -138,7 +125,7 @@
final int str = ((Integer) strength).intValue();
- if (INFO)
+ if (log.isInfoEnabled())
log.info("strength=" + str);
collator.setStrength(str);
@@ -147,7 +134,7 @@
StrengthEnum str = (StrengthEnum) strength;
- if (INFO)
+ if (log.isInfoEnabled())
log.info("strength=" + str);
switch (str) {
@@ -184,7 +171,7 @@
if (mode != null) {
- if(INFO) log.info("mode="+mode);
+ if(log.isInfoEnabled()) log.info("mode="+mode);
switch (mode) {
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java 2010-07-25 20:26:13 UTC (rev 3284)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java 2010-07-25 21:33:21 UTC (rev 3285)
@@ -56,8 +56,8 @@
}
- public JDKSortKeyGenerator(Locale locale, Object strength,
- DecompositionEnum mode) {
+ public JDKSortKeyGenerator(final Locale locale, final Object strength,
+ final DecompositionEnum mode) {
if (locale == null)
throw new IllegalArgumentException();
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-25 20:26:13 UTC (rev 3284)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-25 21:33:21 UTC (rev 3285)
@@ -179,9 +179,9 @@
* The buffer reference is used directly rather than making a
* copy of the data.
*/
- protected KeyBuilder(UnicodeSortKeyGenerator sortKeyGenerator, int len,
- byte[] buf) {
-
+ protected KeyBuilder(final UnicodeSortKeyGenerator sortKeyGenerator,
+ final int len, final byte[] buf) {
+
if (len < 0)
throw new IllegalArgumentException("len");
@@ -349,7 +349,7 @@
* The object responsible for generating sort keys from Unicode strings.
*
* The {@link UnicodeSortKeyGenerator} -or- <code>null</code> if Unicode
- * is not supported by this {@link KeyBuilder} instance.
+ * is not supported by this {@link IKeyBuilder} instance.
*/
final public UnicodeSortKeyGenerator getSortKeyGenerator() {
@@ -390,11 +390,11 @@
public KeyBuilder appendASCII(final String s) {
- int len = s.length();
+ int tmpLen = s.length();
- ensureFree(len);
+ ensureFree(tmpLen);
- for(int j=0; j<len; j++) {
+ for(int j=0; j<tmpLen; j++) {
char ch = s.charAt(j);
@@ -1420,8 +1420,8 @@
}
- static final byte eos = decodeByte(0);
- static final byte negSign = decodeByte(-1);
+ private static final byte eos = decodeByte(0);
+ private static final byte negSign = decodeByte(-1);
// FIXME decodeBigDecimal(int, byte[])
static public BigDecimal decodeBigDecimal(final int offset, final byte[] key) {
@@ -1469,6 +1469,11 @@
return new String(chrs);
}
+ /**
+ * Create an instance for ASCII keys.
+ *
+ * @return The new instance.
+ */
public static IKeyBuilder newInstance() {
return newInstance(DEFAULT_INITIAL_CAPACITY);
@@ -1506,10 +1511,8 @@
/**
* Optional property specifies the library that will be used to generate
- * sort keys from Unicode data. The default always supports Unicode, but
- * the library choice depends on whether or not ICU library is found on
- * the classpath. When the ICU library is present, it is the default.
- * Otherwise the JDK library is the default. You may explicitly specify
+ * sort keys from Unicode data. The ICU library is the default.
+ * You may explicitly specify
* the library choice using one of the {@link CollatorEnum} values. The
* {@link CollatorEnum#ASCII} value may be used to disable Unicode
* support entirely, treating the characters as if they were ASCII. If
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-07-25 20:26:13 UTC (rev 3284)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-07-25 21:33:21 UTC (rev 3285)
@@ -149,6 +149,7 @@
// allow client requests to finish normally.
new ShutdownHelper(threadPool, 10L/*logTimeout*/, TimeUnit.SECONDS) {
+ @Override
public void logTimeout() {
log.warn("Awaiting thread pool termination: elapsed="
@@ -170,6 +171,7 @@
new ShutdownHelper(scheduledExecutorService, 10L/* logTimeout */,
TimeUnit.SECONDS) {
+ @Override
public void logTimeout() {
log.warn("Awaiting sample service termination: elapsed="
@@ -644,12 +646,7 @@
TimeUnit.MILLISECONDS // unit
);
- addScheduledTask(//
- new StartDeferredTasksTask(),// task to run.
- 150, // initialDelay (ms)
- 2000, // delay
- TimeUnit.MILLISECONDS // unit
- );
+ getExecutorService().execute(new StartDeferredTasksTask());
// Setup locator.
resourceLocator = new DefaultResourceLocator(this,
@@ -880,12 +877,12 @@
*/
public boolean isServiceReady() {
- final AbstractClient<T> client = this.client;
+ final AbstractClient<T> thisClient = this.client;
- if (client == null)
+ if (thisClient == null)
return false;
- final IFederationDelegate<T> delegate = client.getDelegate();
+ final IFederationDelegate<T> delegate = thisClient.getDelegate();
if (delegate == null)
return false;
@@ -955,11 +952,11 @@
}
// @todo really, we should test like this everywhere.
- final AbstractClient client = this.client;
+ final AbstractClient thisClient = this.client;
- if (client != null && client.isConnected()) {
+ if (thisClient != null && thisClient.isConnected()) {
- client.getDelegate().serviceLeave(serviceUUID);
+ thisClient.getDelegate().serviceLeave(serviceUUID);
}
@@ -988,7 +985,7 @@
static private String ERR_SERVICE_NOT_READY = "Service is not ready yet.";
/**
- * This task runs periodically. Once {@link #getServiceUUID()} reports a
+ * This task runs once. Once {@link #getServiceUUID()} reports a
* non-<code>null</code> value, it will start an (optional)
* {@link AbstractStatisticsCollector}, an (optional) httpd service, and
* the (required) {@link ReportTask}.
@@ -998,8 +995,6 @@
* {@link ILoadBalancerService} know which services exist, which is
* important for some of its functions.
* <p>
- * Once these task(s) have been started, this task will throw an exception
- * in order to prevent it from being re-executed.
*
* FIXME This should explicitly await jini registrar discovery, zookeeper
* client connected, and whatever other preconditions must be statisified
@@ -1022,8 +1017,7 @@
*/
final long begin = System.currentTimeMillis();
- public StartDeferredTasksTask() {
-
+ private StartDeferredTasksTask() {
}
/**
@@ -1033,8 +1027,6 @@
*/
public void run() {
- final boolean started;
-
try {
// /*
@@ -1051,9 +1043,7 @@
// return;
//
// }
-
- started = startDeferredTasks();
-
+ startDeferredTasks();
} catch (Throwable t) {
log.warn("Problem in report task?", t);
@@ -1062,60 +1052,56 @@
}
- if (started) {
-
- /*
- * Note: This exception is thrown once this task has executed
- * successfully.
- */
-
- throw new RuntimeException("Normal completion.");
-
- }
-
}
/**
* Starts performance counter collection once the service {@link UUID}
* is known.
*
- * @return <code>true</code> iff performance counter collection was
- * started.
- *
* @throws IOException
* if {@link IDataService#getServiceUUID()} throws this
* exception (it never should since it is a local method
* call).
*/
- protected boolean startDeferredTasks() throws IOException {
+ protected void startDeferredTasks() throws IOException {
// elapsed time since we started running this task.
final long elapsed = System.currentTimeMillis() - begin;
- if (getServiceUUID() == null) {
-
+ // Wait for the service ID to become available, trying every
+ // two seconds, while logging failures.
+ while (true) {
+ if (getServiceUUID() != null) {
+ break;
+ }
if (elapsed > 1000 * 10)
log.warn(ERR_NO_SERVICE_UUID + " : iface="
+ getServiceIface() + ", name=" + getServiceName()
+ ", elapsed=" + elapsed);
else if (log.isInfoEnabled())
- log.info(ERR_NO_SERVICE_UUID);
-
- return false;
-
+ log.info(ERR_NO_SERVICE_UUID);
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ }
}
- if (!isServiceReady()) {
-
+ // Wait for the service to become ready, trying every
+ // two seconds, while logging failures.
+ while (true) {
+ if (isServiceReady()) {
+ break;
+ }
if (elapsed > 1000 * 10)
log.warn(ERR_SERVICE_NOT_READY + " : iface="
+ getServiceIface() + ", name=" + getServiceName()
+ ", elapsed=" + elapsed);
else if (log.isInfoEnabled())
log.info(ERR_SERVICE_NOT_READY + " : " + elapsed);
-
- return false;
-
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ }
}
/*
@@ -1143,9 +1129,6 @@
// notify delegates that deferred startup has occurred.
AbstractFederation.this.didStart();
-
- return true;
-
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java 2010-07-25 20:26:13 UTC (rev 3284)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java 2010-07-25 21:33:21 UTC (rev 3285)
@@ -32,8 +32,12 @@
import org.apache.log4j.Logger;
+import com.bigdata.btree.DefaultTupleSerializer;
import com.bigdata.btree.IIndex;
import com.bigdata.btree.IndexMetadata;
+import com.bigdata.btree.keys.ASCIIKeyBuilderFactory;
+import com.bigdata.btree.keys.CollatorEnum;
+import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.journal.IIndexManager;
import com.bigdata.journal.ITx;
import com.bigdata.journal.TimestampUtility;
@@ -99,6 +103,25 @@
indexMetadata
.setSplitHandler(LogicalRowSplitHandler.INSTANCE);
+ if (CollatorEnum.JDK.toString().equals(
+ System.getProperty(KeyBuilder.Options.COLLATOR))) {
+ /*
+ * The JDK RulesBasedCollator embeds nul bytes in the
+ * Unicode sort keys. This makes them unsuitable for the
+ * SparseRowStore, which can not locate the start of the
+ * column name if there are embedded nuls in a Unicode
+ * primary key. As a work around, this forces an ASCII
+ * collation sequence if the JDK collator is the
+ * default. This is not ideal since non-ascii
+ * distinctions will be lost, but it is better than
+ * being unable to decode the column names.
+ */
+ log.warn("Forcing ASCII collator.");
+ indexMetadata
+ .setTupleSerializer(new DefaultTupleSerializer(
+ new ASCIIKeyBuilderFactory()));
+ }
+
// Register the index.
indexManager.registerIndex(indexMetadata);
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java 2010-07-25 20:26:13 UTC (rev 3284)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java 2010-07-25 21:33:21 UTC (rev 3285)
@@ -49,17 +49,22 @@
* prefix.
* <p>
* The encoded schema name is followed by the {@link KeyType#getByteCode()} and
- * then by a <code>nul</code> byte. By searching for the <code>nul</code>
- * byte we can identify the end of the encoded schema name and also the data
- * type of the primary key. Most kinds of primary keys have a fixed length
- * encoding, e.g., {@link Long}, {@link Double}, etc. However, Unicode primary
- * keys have a variable length encoding which makes life more ... complex. Since
- * the keys need to reflect the total sort order we can not include the byte
- * count of the primary key in the key itself. The only reasonable approach is
- * to append a byte sequence to the key that never occurs within the generated
- * Unicode sort keys. We use a <code>nul</code> byte for this purpose since it
- * is not emitted by most Unicode collation implementations as it would cause
- * grief for C-language strings.
+ * then by a <code>nul</code> byte. By searching for the <code>nul</code> byte
+ * we can identify the end of the encoded schema name and also the data type of
+ * the primary key. Most kinds of primary keys have a fixed length encoding,
+ * e.g., {@link Long}, {@link Double}, etc.
+ * <p>
+ * Unicode primary keys have a variable length encoding which makes life more
+ * complex. For Unicode primary keys, we break with the collation order and use
+ * the UTF8 encoding of the key. This means that the primary key can be decoded
+ * and preserves hierarchical namespace clustering within the row store but does
+ * not impose a total sort order per Unicode sort key semantics. The only
+ * reasonable approach is to append a byte sequence to the key that never occurs
+ * within the generated Unicode sort keys. Again, we use a <code>nul</code> byte
+ * to mark the end of the Unicode primary key since it is not emitted by most
+ * Unicode collation implementations as it would cause grief for C-language
+ * strings. (However, see SparseRowStore.Options#PRIMARY_KEY_UNICODE_CLEAN} for
+ * information on backward compatibility.)
*
* @see Schema#fromKey(IKeyBuilder, Object)
* @see KeyType#getKeyType(byte)
@@ -166,15 +171,15 @@
* The decoded primary key.
*
* @throws UnsupportedOperationException
- * if the primary key can not be decoded (e.g., for
- * {@link KeyType#Unicode} keys).
+ * if the primary key can not be decoded.
*/
public Object getPrimaryKey() {
if(primaryKey == null) {
-
- throw new UnsupportedOperationException("Can not decode: keyType="+primaryKeyType);
-
+
+ throw new UnsupportedOperationException("Can not decode: keyType="
+ + primaryKeyType);
+
}
return primaryKey;
@@ -220,14 +225,13 @@
* Note: the KeyType byte occurs after the schema name bytes and before
* the [nul].
*/
+ int primaryKeyOffset = 0;
{
boolean found = false;
int schemaBytesLength = 0;
- int primaryKeyOffset = 0;
-
for (int i = 0; i < key.length; i++) {
if (key[i] == (byte) 0) {
@@ -256,7 +260,6 @@
this.primaryKeyTypeOffset = schemaBytesLength;
- this.primaryKeyOffset = primaryKeyOffset;
// note: ArrayIndexOutOfBounds with index==-1 means ICU library not on classpath!
this.primaryKeyType = KeyType.getKeyType(KeyBuilder.decodeByte(key[primaryKeyTypeOffset]));
@@ -273,42 +276,48 @@
primaryKeyLength = primaryKeyType.getEncodedLength();
+ this.primaryKeyOffset = primaryKeyOffset;
+
columnNameOffset = primaryKeyOffset + primaryKeyLength;
} else {
/*
- * Scan for the next [nul] byte.
+ * Scan for the next [nul] byte (ASCII).
*/
boolean found = false;
int primaryKeyLength = 0;
-
+
for (int i = primaryKeyOffset; i < key.length; i++) {
if (key[i] == (byte) 0) {
primaryKeyLength = i - primaryKeyOffset;
-
+
found = true;
-
+
break;
-
+
}
}
-
- if(!found) {
+ if (!found) {
+
throw new RuntimeException(
"Could not locate the end of the encoded schema name: keyType="
- + primaryKeyType+", key="+BytesUtil.toString(key));
+ + primaryKeyType + ", key="
+ + BytesUtil.toString(key));
}
this.primaryKeyLength = primaryKeyLength;
- // Note: also skips the [nul] byte terminating the primary key.
+ this.primaryKeyOffset = primaryKeyOffset;
+
+ // Note: also skips the [nul] byte terminating the primary
+ // key.
this.columnNameOffset = primaryKeyOffset + primaryKeyLength + 1;
}
@@ -327,10 +336,26 @@
primaryKey = KeyBuilder.decodeFloat(key, primaryKeyOffset);
break;
case Unicode:
- /*
- * Note: Decode is not possible for this case.
- */
- primaryKey = null;
+ if (SparseRowStore.primaryKeyUnicodeClean) {
+ final byte[] bytes = new byte[primaryKeyLength];
+ System.arraycopy(key, primaryKeyOffset, bytes, 0, primaryKeyLength);
+ try {
+ primaryKey = new String(bytes, SparseRowStore.UTF8);
+ } catch (UnsupportedEncodingException ex) {
+ throw new RuntimeException(
+ "Could not decode the primary key"
+ + ": primaryKeyOffset="
+ + primaryKeyOffset
+ + ", primaryKeyLength="
+ + primaryKeyLength + ", key="
+ + BytesUtil.toString(key));
+ }
+ } else {
+ /*
+ * Note: Decode is not possible for this case.
+ */
+ primaryKey = null;
+ }
break;
case ASCII:
primaryKey = KeyBuilder.decodeASCII(key, primaryKeyOffset,
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/Schema.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-25 20:26:13 UTC (rev 3284)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-25 21:33:21 UTC (rev 3285)
@@ -154,7 +154,7 @@
/*
* Key builder stuff.
*/
-
+
/**
* Helper method appends a typed value to the compound key (this is used to
* get the primary key into the compound key).
@@ -172,8 +172,9 @@
*
* @see KeyDecoder
*/
- final protected IKeyBuilder appendPrimaryKey(IKeyBuilder keyBuilder, Object v, boolean successor) {
-
+ final protected IKeyBuilder appendPrimaryKey(final IKeyBuilder keyBuilder,
+ final Object v, final boolean successor) {
+
final KeyType keyType = getPrimaryKeyType();
if (successor) {
@@ -187,9 +188,27 @@
case Float:
return keyBuilder.append(successor(keyBuilder,((Number) v).floatValue()));
case Double:
- return keyBuilder.append(successor(keyBuilder,((Number) v).doubleValue()));
- case Unicode:
- return keyBuilder.appendText(v.toString(), true/*unicode*/, true/*successor*/).appendNul();
+ return keyBuilder.append(successor(keyBuilder, ((Number) v)
+ .doubleValue()));
+ case Unicode: {
+ final String tmp = v.toString();
+ if (SparseRowStore.primaryKeyUnicodeClean) {
+ try {
+ keyBuilder.append(
+ SuccessorUtil.successor(tmp
+ .getBytes(SparseRowStore.UTF8)))
+ .appendNul();
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ // primary key in backwards compatibility mode.
+ keyBuilder
+ .appendText(tmp, true/* unicode */, true/* successor */)
+ .appendNul();
+ }
+ return keyBuilder;
+ }
case ASCII:
return keyBuilder.appendText(v.toString(), false/*unicode*/, true/*successor*/).appendNul();
case Date:
@@ -212,8 +231,22 @@
return keyBuilder.append(((Number) v).floatValue());
case Double:
return keyBuilder.append(((Number) v).doubleValue());
- case Unicode:
- return keyBuilder.appendText(v.toString(),true/*unicode*/,false/*successor*/).appendNul();
+ case Unicode: {
+ final String tmp = v.toString();
+ if (SparseRowStore.primaryKeyUnicodeClean) {
+ try {
+ keyBuilder.append(tmp.getBytes(SparseRowStore.UTF8))
+ .appendNul();
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ // primary key in backwards compatibility mode.
+ keyBuilder.appendText(v.toString(), true/* unicode */,
+ false/* successor */).appendNul();
+ }
+ return keyBuilder;
+ }
case ASCII:
return keyBuilder.appendText(v.toString(),false/*unicode*/,false/*successor*/).appendNul();
case Date:
@@ -243,8 +276,8 @@
* {@link #toKey(Object)}, which correctly forms the successor
* key in all cases.
*/
- final private Object successor(IKeyBuilder keyBuilder,Object v) {
-
+ final private Object successor(final IKeyBuilder keyBuilder, final Object v) {
+
final KeyType keyType = getPrimaryKeyType();
switch(keyType) {
@@ -290,7 +323,8 @@
*
* @see KeyDecoder
*/
- final protected IKeyBuilder fromKey(IKeyBuilder keyBuilder,Object primaryKey) {
+ final protected IKeyBuilder fromKey(final IKeyBuilder keyBuilder,
+ final Object primaryKey) {
keyBuilder.reset();
@@ -319,8 +353,9 @@
*
* @return
*/
- final public byte[] getPrefix(IKeyBuilder keyBuilder,Object primaryKey) {
-
+ final public byte[] getPrefix(final IKeyBuilder keyBuilder,
+ final Object primaryKey) {
+
return fromKey(keyBuilder, primaryKey).getKey();
}
@@ -384,8 +419,8 @@
* @throws IllegalArgumentException
* if <i>col</i> is not valid as the name of a column.
*/
- public byte[] getKey(IKeyBuilder keyBuilder, Object primaryKey, String col,
- long timestamp) {
+ public byte[] getKey(final IKeyBuilder keyBuilder, final Object primaryKey,
+ final String col, final long timestamp) {
if (keyBuilder == null)
throw new IllegalArgumentException();
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java 2010-07-25 20:26:13 UTC (rev 3284)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java 2010-07-25 21:33:21 UTC (rev 3285)
@@ -24,6 +24,7 @@
*/
package com.bigdata.sparse;
+import java.text.RuleBasedCollator;
import java.util.Iterator;
import java.util.Map;
@@ -33,7 +34,9 @@
import com.bigdata.btree.IIndex;
import com.bigdata.btree.IRangeQuery;
import com.bigdata.btree.ITuple;
+import com.bigdata.btree.IndexMetadata;
import com.bigdata.btree.filter.FilterConstructor;
+import com.bigdata.btree.keys.CollatorEnum;
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.journal.ITimestampService;
@@ -186,6 +189,17 @@
* md.setSplitHandler(LogicalRowSplitHandler.INSTANCE);
* </pre>
*
+ * Note: The JDK {@link RuleBasedCollator} embeds <code>nul</code> bytes in
+ * the Unicode sort keys. This makes them unsuitable for the row store which
+ * can not locate the start of the column name if there are embedded
+ * <code>nul</code>s in the primaryKey. Therefore, if you are using the
+ * {@link CollatorEnum#JDK} as your default collator, then you MUST override
+ * the {@link IndexMetadata} for the row store to use either an ASCII
+ * collator or the ICU collator. In general, the ICU collator is superior to
+ * the JDK collator and will be used by default. The ASCII collator is not
+ * ideal since non-ascii distinctions will be lost, but it is better than
+ * being unable to decode the data in the row store.
+ *
* @param ndx
* The index.
*/
@@ -1029,4 +1043,53 @@
}
+ /**
+ * Options for the {@link SparseRowStore}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan
+ * Thompson</a>
+ * @version $Id$
+ */
+ public interface Options {
+
+ /**
+ * The primary key was originally written using a Unicode sort key.
+ * However, the JDK generates Unicode sort keys with embedded nuls and
+ * that broke the logic to detect the end of the Unicode primary keys.
+ * In order to accommodate this behavior, the Unicode primary key is now
+ * encoded as UTF8 which also has the advantage that we can decode
+ * Unicode primary keys. Standard prefix compression on the B+Tree
+ * should make up for the larger representation of the Unicode primary
+ * key in the B+Tree.
+ * <p>
+ * This change was introduced on 7/15/2010 in the trunk and breaks
+ * compatibility with earlier revisions of the {@link SparseRowStore}.
+ * This flag may be set to <code>false</code> for backward
+ * compatibility.
+ *
+ * @see Options#DEFAULT_PRIMARY_KEY_UNICODE_CLEAN
+ */
+ String PRIMARY_KEY_UNICODE_CLEAN = Schema.class.getName()
+ + ".primaryKey.unicodeClean";
+
+ /**
+ * FIXME Change over the [true] for the next release.
+ *
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/107
+ */
+ String DEFAULT_PRIMARY_KEY_UNICODE_CLEAN = "false";
+
+ }
+
+ /**
+ * This is a global option since it was always <code>false</code> for
+ * historical stores.
+ *
+ * @see Options#PRIMARY_KEY_UNICODE_CLEAN
+ */
+ final static transient boolean primaryKeyUnicodeClean = Boolean
+ .valueOf(System.getProperty(
+ SparseRowStore.Options.PRIMARY_KEY_UNICODE_CLEAN,
+ SparseRowStore.Options.DEFAULT_PRIMARY_KEY_UNICODE_CLEAN));
+
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/util/config/NicUtil.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/util/config/NicUtil.java 2010-07-25 20:26:13 UTC (rev 3284)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/util/config/NicUtil.java 2010-07-25 21:33:21 UTC (rev 3285)
@@ -33,7 +33,9 @@
import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.UnknownHostException;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Enumeration;
import java.util.Collections;
import java.util.logging.LogRecord;
@@ -102,7 +104,6 @@
{
NetworkInterface nic = NetworkInterface.getByName(name);
if (nic == null) {
-
// try by IP address
InetAddress targetIp = null;
try {
@@ -116,6 +117,43 @@
}
/**
+ * Method that returns a <code>Map</code> in which the key component
+ * of each element is one of the addresses of one of the network
+ * interface cards (nics) installed on the current node, and the
+ * corresponding value component is the name of the associated
+ * nic to which that address is assigned.
+ *
+ * @return a <code>Map</code> of key-value pairs in which the key
+ * is an instance of <code>InetAddress</code> referencing
+ * the address of one of the network interface cards installed
+ * on the current node, and the corresponding value is a
+ * <code>String</code> referencing the name of the associated
+ * network interface to which the address is assigned.
+ *
+ * @throws SocketException if there is an error in the underlying
+ * I/O subsystem and/or protocol.
+ */
+ public static Map<InetAddress, String> getInetAddressMap()
+ throws SocketException
+ {
+ Map<InetAddress, String> retMap = new HashMap<InetAddress, String>();
+
+ //get all nics on the current node
+ Enumeration<NetworkInterface> nics =
+ NetworkInterface.getNetworkInterfaces();
+
+ while( nics.hasMoreElements() ) {
+ NetworkInterface curNic = nics.nextElement();
+ Enumeration<InetAddress> curNicAddrs = curNic.getInetAddresses();
+ String curNicName = curNic.getName();
+ while( curNicAddrs.hasMoreElements() ) {
+ retMap.put( curNicAddrs.nextElement(), curNicName );
+ }
+ }
+ return retMap;
+ }
+
+ /**
* Method that searches for and returns an array whose elements
* are all the network interface(s) that correspond to the specified
* <code>name</code>.
@@ -467,6 +505,253 @@
}
/**
+ * Special-purpose convenience method that returns a
+ * <code>String</code> value representing the ip address of
+ * the current node; where the value that is returned is
+ * determined according to following criteria:
+ * <p>
+ * <ul>
+ * <li> If a non-<code>null</code> value is input for the
+ * <code>systemPropertyName</code> parameter, then
+ * this is viewed as a declaration by the caller that
+ * that the system property with that given value
+ * should take precedence over all other means of
+ * determining the desired ip address. As such, this
+ * method determines if a system property having the
+ * given has indeed been set and, if it has, returns
+ * the ip address of the nic having that name; or
+ * <code>null</code> if there is no nic with the
+ * desired name installed on the node.
+ * <li> If a non-<code>null</code> value is input for the
+ * <code>systemPropertyName</code> parameter, but
+ * no system property with that name has been set,
+ * and <code>true</code> has been passed in for
+ * the <code>fallbackOk</code> parameter, then this
+ * method will return the IPV4 based address of the
+ * first reachable nic that is found on the node.
+ * Upon failing to find such an address, if the
+ * <code>loopbackOk</code> parameter is also
+ * <code>true</code>, then this method will return
+ * the <i>loop back</i> address of the node; otherwise
+ * <code>null</code> is returned.
+ * <li> If <code>null</code> is input for the
+ * <code>systemPropertyName</code> parameter, but
+ * a non-<code>null</code> value is input for the
+ * <code>defaultNic</code> parameter, then this
+ * method returns the ip address of the nic having
+ * that name; or <code>null</code> if there is no
+ * nic with the desired default name installed on the
+ * node.
+ * <li> If <code>null</code> is input for both the
+ * <code>systemPropertyName</code> parameter and the
+ * <code>defaultNic</code> parameter, and if the
+ * <code>fallbackOk</code> parameter is <code>true</code>,
+ * then this method will return the IPV4 based address
+ * of the first reachable nic that is found on the node.
+ * Upon failing to find such an address, if the
+ * <code>loopbackOk</code> parameter is also
+ * <code>true</code>, then this method will return
+ * the <i>loop back</i> address of the node; otherwise
+ * <code>null</code> is returned.
+ * </ul>
+ * <p>
+ * This method can be called from within a configuration
+ * as well as from within program control.
+ *
+ * @param systemPropertyName <code>String</code> value containing
+ * the name of a system property whose
+ * value is the network interface name
+ * whose ip address should be returned.
+ * May be <code>null</code>.
+ *
+ * @param defaultNic <code>String</code> value containing
+ * the name of the network interface
+ * whose ip address should be returned
+ * if <code>null</code> is input for the
+ * <code>systemPropertyName</code> parameter.
+ *
+ * @param fallbackOk if <code>true</code>, then if either
+ * no system property is set having the
+ * name referenced by the
+ * <code>systemPropertyName</code> parameter,
+ * or if <code>null</code> is input for both
+ * the <code>systemPropertyName</code>
+ * parameter and the <code>defaultNic</code>
+ * parameter, return the IPV4 based address
+ * of the first reachable network interface
+ * that can be found on the node.
+ *
+ * @param loopbackOk if <code>true</code>, and if <code>true</code>
+ * is also input for the <code>fallbackOk</code>
+ * parameter, then if this method attempts,
+ * but fails, to find a valid IPV4 fallback
+ * address, then the node's <i>loop back</i>
+ * address is returned.
+ *
+ * @return a <code>String</code> representing an ip address associated
+ * with the current node; where the value that is returned is
+ * determined according to the criteria described above.
+ */
+ public static String getIpAddress(String systemPropertyName,
+ String defaultNic,
+ boolean fallbackOk,
+ boolean loopbackOk)
+ throws SocketException, IOException
+ {
+ if(systemPropertyName != null) {//system property takes precedence
+ String nicName = System.getProperty(systemPropertyName);
+ boolean propSet = true;
+ if(nicName == null) {
+ propSet = false;
+ } else {
+ // handle ant script case where the system property
+ // may not have been set on the command line, but
+ // was still set to "${<systemPropertyName>}" using
+ // ant <sysproperty> tag
+ String rawProp = "${" + systemPropertyName + "}";
+ if( rawProp.equals(nicName) ) propSet = false;
+ }
+ if(propSet) {
+ return getIpAddress(nicName, 0, loopbackOk);
+ } else {//desired system property not set, try fallback
+ if(fallbackOk) {
+ return getDefaultIpv4Address(loopbackOk);
+ } else {
+ return null;
+ }
+ }
+ } else {//no system property name provided, try default nic
+ if(defaultNic != null) {
+ return getIpAddress(defaultNic, 0, loopbackOk);
+ } else {//no default nic provided, try fallback
+ if(fallbackOk) {
+ return getDefaultIpv4Address(loopbackOk);
+ } else {
+ return null;
+ }
+ }
+ }
+ }
+
+ public static String getIpAddress()
+ throws SocketException, IOException
+ {
+ return getIpAddress(null, null, true, true);
+ }
+
+ /**
+ * Examines each address associated with each network interface
+ * card (nic) installed on the current node, and returns the
+ * <code>String</code> value of the first such address that is
+ * determined to be both <i>reachable</i> and an address type
+ * that represents an <i>IPv4</i> address.
+ *
+ * This method will always first examine addresses that are
+ * <i>not</i> the <i>loopback</i> address (<i>local host</i>);
+ * returning a loopback adddress only if <code>true</code>
+ * is input for the <code>loopbackOk</code> parameter, and
+ * none of the non-loopback addresses satisfy this method's
+ * search criteria.
+ *
+ * If this method fails to find any address that satisfies the
+ * above criteria, then this method returns <code>null</code>.
+ *
+ * @param loopbackOk if <code>true</code>, then upon failure
+ * find an non-<i>loopback</i> address that
+ * satisfies this method's search criteria
+ * (an IPv4 type address and reachable), the
+ * first loopback address that is found to be
+ * reachable is returned.
+ *
+ * If <code>false</code> is input for this
+ * parameter, then this method will examine
+ * only those addresses that do <i>not</i>
+ * correspond to the corresponding nic's
+ * loopback address.
+ *
+ * @return a <code>String</code> value representing the first
+ * network interface address installed on the current
+ * node that is determined to be both <i>reachable</i>
+ * and an IPv4 type address; where the return value
+ * corresponds to a <i>loopback</i> address only if
+ * <code>true</code> is input for the <code>loopbackOk</code>
+ * parameter, and no non-loopback address satisfying
+ * the desired criteria can be found. If this method
+ * fails to find any address that satisfies the desired
+ * criteria, then <code>null</code> is returned.
+ *
+ * @throws SocketException if there is an error in the underlying
+ * I/O subsystem and/or protocol while retrieving the
+ * the network interfaces currently installed on the
+ * node.
+ *
+ * @throws IOException if a network error occurs while determining
+ * if a candidate return address is <i>reachable</i>.
+ */
+ public static String getDefaultIpv4Address(boolean loopbackOk)
+ throws SocketException, IOException
+ {
+ //get all nics on the current node
+ Enumeration<NetworkInterface> nics =
+ NetworkInterface.getNetworkInterfaces();
+ while( nics.hasMoreElements() ) {
+ NetworkInterface curNic = nics.nextElement();
+ List<InterfaceAddress> interfaceAddrs =
+ curNic.getInterfaceAddresses();
+ for(InterfaceAddress interfaceAddr : interfaceAddrs) {
+ InetAddress inetAddr = interfaceAddr.getAddress();
+ boolean isIpv4 = inetAddr instanceof Inet4Address;
+ boolean isLoopbackAddress = inetAddr.isLoopbackAddress();
+ if(isIpv4) {
+ if(isLoopbackAddress) continue;
+ boolean isReachable = inetAddr.isReachable(3*1000);
+ Inet4Address inet4Addr = (Inet4Address)inetAddr;
+ String retVal = inet4Addr.getHostAddress();
+
+ jiniConfigLogger.log
+ (CONFIG, "default IPv4 address: "+retVal);
+ utilLogger.log
+ (Level.TRACE, "default IPv4 address: "+retVal);
+ return retVal;
+ }
+ }
+ }
+
+ if(!loopbackOk) return null;
+
+ nics = NetworkInterface.getNetworkInterfaces();
+ while( nics.hasMoreElements() ) {
+ NetworkInterface curNic = nics.nextElement();
+ List<InterfaceAddress> interfaceAddrs =
+ curNic.getInterfaceAddresses();
+ for(InterfaceAddress interfaceAddr : interfaceAddrs) {
+ InetAddress inetAddr = interfaceAddr.getAddress();
+ boolean isIpv4 = inetAddr instanceof Inet4Address;
+ boolean isLoopbackAddress = inetAddr.isLoopbackAddress();
+ if(isIpv4) {
+ if(!isLoopbackAddress) continue;
+ boolean isReachable = inetAddr.isReachable(3*1000);
+ Inet4Address inet4Addr = (Inet4Address)inetAddr;
+ String retVal = inet4Addr.getHostAddress();
+
+ jiniConfigLogger.log
+ (CONFIG, "default IPv4 address: "+retVal);
+ utilLogger.log
+ (Level.TRACE, "default IPv4 address: "+retVal);
+ return retVal;
+ }
+ }
+ }
+ return null;
+ }
+
+ public static String getDefaultIpv4Address()
+ throws SocketException, IOException
+ {
+ return getDefaultIpv4Address(false);//localhost NOT ok
+ }
+
+ /**
* Intended for use by scripts.
*/
public static void main(String[] args) {
Deleted: branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/AbstractUnicodeKeyBuilderTestCase.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/AbstractUnicodeKeyBuilderTestCase.java 2010-07-25 20:26:13 UTC (rev 3284)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/btree/AbstractUnicodeKeyBuilderTestCase.java 2010-07-25 21:33:21 UTC (rev 3285)
@@ -1,183 +0,0 @@
-/*
-
-Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-*/
-/*
- * Created on Nov 29, 2007
- */
-
-package com.bigdata.btree;
-
-import java.util.Locale;
-import java.util.Properties;
-
-import junit.framework.TestCase2;
-
-import com.bigdata.btree.keys.DefaultKeyBuilderFactory;
-import com.bigdata.btree.keys.IKeyBuilder;
-import com.bigdata.btree.keys.KeyBuilder;
-import com.bigdata.btree.keys.StrengthEnum;
-import com.bigdata.btree.keys.KeyBuilder.Options;
-
-/**
- * Base class for the test suites that examine support for Unicode sort keys in
- * {@link KeyBuilder}.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- */
-abstract public class AbstractUnicodeKeyBuilderTestCase extends TestCase2 {
-
- /**
- *
- */
- public AbstractUnicodeKeyBuilderTestCase() {
- }
-
- /**
- * @param arg0
- */
- public AbstractUnicodeKeyBuilderTestCase(String arg0) {
- super(arg0);
- }
-
- /**
- * Test ability to encode unicode data into a variable length byte[] that
- * allows direct byte-by-byte comparisons which maintain the local-specific
- * sort order of the original strings.
- */
- public void test_keyBuilder_unicode_string_key_us_primary() {
-
- /*
- * Setup an instance for US English with strength := PRIMARY.
- */
-
- Pr...
[truncated message content] |
|
From: <tho...@us...> - 2010-07-25 20:26:20
|
Revision: 3284
http://bigdata.svn.sourceforge.net/bigdata/?rev=3284&view=rev
Author: thompsonbry
Date: 2010-07-25 20:26:13 +0000 (Sun, 25 Jul 2010)
Log Message:
-----------
Removed the notes file used to drive this refactor.
Removed Paths:
-------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/LexiconRefactor.txt
Deleted: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/LexiconRefactor.txt
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/LexiconRefactor.txt 2010-07-23 22:21:55 UTC (rev 3283)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/LexiconRefactor.txt 2010-07-25 20:26:13 UTC (rev 3284)
@@ -1,415 +0,0 @@
-These are my notes on the lexicon refactor.
-
- #5. Order preserving coding in the statement indices together
- with less data stored in the lexicon and exploiting that
- information in FILTERs (range queries, value tests, etc).
- This should be a lexicon option in case people want to have
- exact lexical values preserved. The data type would be
- preserved so an xsd:int and an xsd:long would not compare
- as EQ unless they were coerced to the same data type during
- query (such coercion would require running against the
- value range for each source data type convertable to the
- target data type).
-
- There are complex issues involving xsd:decimal (BigDecimal)
- and xsd:integer (BigInteger). These have very large
- ranges. Making them into inline values would require a
- variable length component for the "termId", which in turn
- makes it more complex to form and decode the statement
- index keys.
-
- Long literals should be supported as transparently
- compressed blobs to get the data out of the index. This
- raises the question of just where to store those
- blobs. This is the same issue for BFS or key-blob stores.
- Basically, we need to dump them into the local file system
- in a directory structure parallel to the journals and
- segments. If we move a shard, we also need to move the
- blobs. Note that there are relatively low limits on the
- #of files/subdirectories in a directory for many operating
- systems.
-
- - done. Verify key component decode with variable length
- keys.
-
- - Add smart separator key selection for the ID2TERM index?
-
- - Note: We can not inline plain text or language code
- literals unless the ASCII option was specified. Ditto
- for URIs. This is because we can not decode Unicode sort
- keys. An extensibility mechanism has been defined to
- handle this using "enumerations".
-
- - Use raw records and an IOverflowHandler to move large
- literals out of the ID2TERM index. Likewise, we should
- probably not register large literals in the TERM2ID
- index. However, they could still be indexed, e.g., free
- text indexing or XML literal indexing. [One problem with
- not entering them into the TERM2ID index is that we can
- get duplicate entries. Perhaps we should enter the MD5
- signature of the long literal, which is 128 bytes, as the
- key. Those keys should all be in their own prefix space
- for long literals.]
-
- - Take notes on the changes to the physical schema in order
- to support a port of data. However, the easiest way to
- do a port is to not break the existing schema and allow
- people to export/import into a different triple store
- relation in the same database. Even easier is export /
- import into a new database instance.
-
- - Generate an unsigned byte[] coding of BigInteger (done)
- and BigDecimal (todo).
-
-=========================
-
-There are several wrinkles to proper handling of datatype literals,
-some of which only apply when we choose to inline those literals into
-the statement indices. In general, this would result in statement
-indices with variable length keys and the SPO class would have to be
-modified to correctly decode datatype literals when they were inlined
-within the key.
-
-The biggest single initial win will be not having to indirect through
-the lexicon to resolve xsd:int, xsd:float, xsd:double, etc. This will
-have a huge impact for aggregations. A secondary win, and something
-which can be phased in over time, is the ability to do range queries
-against datatype values. For example, 50 < AGE < 60. Today, the
-filter is applied after the solutions have been identified. Once we
-inline numeric datatypes into the statement indices (and modulo issues
-surrounding casts and filters) these queries could also be evaluated
-against the desired key-range of the statement indices. This
-optimization can be approached incrementally because the existing
-approach (filtering the solutions) will continue to work correctly.
-
-Issues arise with:
-
-- datatypes which have long representations (xsd:integer,
- xsd:decimal). It appears that we can handle both xsd:integer and
- xsd:decimal inline. Therefore it would be reasonable to either
- represent all literals having numeric values in a single value space
- -or- represent all values for each each numeric datatype within its
- own region by using a prefix to separate the keys accordinging to
- the numeric data type. The advantage of the former is we can filter
- by data type in a single scan across all numeric values. The
- advantage of the latter is that we can represent the values more
- tersely since we do not need to convert everything to xsd:decimal
- before encoding it as an unsigned byte[] key component. The second
- approach requires us to query the union of the numeric datatype key
- spaces when a cast is used in the query but will be much faster if
- no cast is used. [We are still looking into the xsd:decimal mapping
- onto keys, but this appears to be simpler than mapping BigDecimal
- since xsd:decimal DOES NOT maintain any information about the
- precision of the value. Representing the precision would require us
- to carry the original scale in the value associated with the tuple
- in the statement index while the key coded the value space.]
-
-- geospatial datatypes. these need to be mapped onto a spatial index.
- the statement would just use the termId of the literal representing
- the point, rectangle, or generalized shape of the extent. Since we
- are assigning a termId in the lexicon, we can add geospatial indices
- later without changing anything in the existing indices.
-
-- datatypes which correspond to durations. I am open to suggestions
- here. This might be handled in a manner similar to spatial data
- types or by using temporal reasoners. Either way, we would store
- the termId in the statement index. We would have to recognize when
- a query was addressing a one-dimensional data type for which we
- lacked a key range index scan and handle it as we handle all queries
- today (by resolving the lexical values from the id2term index and
- then filtering for GT, LT, or EQ).
-
-- Infinately repeating intervals, e.g., the 5th day of each month. We
- can't really code such things into a spatial index since it would
- require an infinite number of bands on the index. I am thinking
- that temporal reasoners are the only real solution here.
-
-- dateTime. There is no XML schema datatype which corresponds to an
- unambiguous timeOnTimeline value. Something similar can be derived
- from xsd:dateTime. In order to also preserve the timezone we would
- have to encode that in the value associated with the statement
- tuple. [Alternatively, we could encode all spatial and/or temporal
- information in spatial indices, in which case the original Literal
- is stored in the id2term index, however the dateTime still needs to
- be unambiguous.]
-
-- Long and very long literals. these should be taken out of the
- id2term index and put into raw record blobs which are then flowed
- into the index segments for scale-out (long literals, e.g., up to a
- few mb) or an addressible file system abstraction (very long
- literals). regardless of how we handle the literal, the statement
- index would use a termId value for the literal. Since some literals
- can not reasonably be represented inline in the statement indices,
- none of the literals can be represented inline and we can not
- perform key-range scans on the statement indices for plain or
- language code literals.
-
-- xml literals. per long and very long literals. one would expect to
- do additional indexing on the XML blobs at some point, but that is
- beyond where we are today.
-
-- extensible data types. I think that there are easy and hard cases
- in this category. Some interesting examples might be custom
- indexing for genomics data. For some kinds of extensibility we
- would probably register additional indices (geospatial).
-
-This will require that Literal#equals(Object) be implemented such that
-representations of the same point in the value space are equals
-without regard to their lexical form. Comparison operators must
-likewise respect the total ordering of the value spaces within each of
-the data type literals (or across all of them if we choose to
-interleave the value spaces for all numeric types, which I am inclined
-NOT to do). Literal#stringValue() might be a lazy method to avoid
-overhead if we wrap an inline literal from a statement index as a
-Literal.
-
-One thing that we should do is mark which datatypes are inline and
-assign unique codes for those data types (reserving either one or two
-bytes for that information and using it to create partitions in the
-key space for the inlined values). This will provide a declarative
-mechanism for the query processor so that it can understand when a
-filter can be mapped onto a key-range scan of a statement index. This
-could be part of the state serialized into the global row store. An
-open question is whether and how to indicate relationships among the
-inline datatypes.
-
-Bryan
-
- ----------
-
-At this time, literals can only appear in the Object position of an
-RDF statement. This limits the scope of the proposed change to the
-ISPO#o() method, the SPO#o public field, and the ISPO#get(int slot)
-method. However, these changes will still cause a lot of distruption
-as we break the assumption that the statement index keys consist
-solely of term identifiers.
-
-Open questions:
-
-- How to represent a statement key component which is not a term
- identifier? Maybe we want to inline only those values which can fit
- into a long and use the high bits to differentiate, interleave all
- the numeric values except for BigInteger and BigDecimal since their
- representations are too long. We can immediately benefit from
- inlining [but this requires an additional byte per object] IF we
- restrict our goals to fast resolution of numeric literals rather
- than range-based query of statement indices. If we also disable
- BigInteger and BigDecimal when values are being inlined key-range
- queries on OSP(C).
-
- We need to create an interface / object for the internal
- representation of the RDF Value, e.g., RDFValueInternal or VI. This
- has been a long termId up to now. With this generalization, it can
- still be a termId (URIs, literals), the termId could be more than 64
- bits (if more partitions could be created), it can directly encode a
- boolean, char, short, int, long, BigInteger, etc. It could even
- encode short literals directly. We need an isTermId() method. When
- false, the RDF Value is expressed inline. When it is inline (not a
- termId), asValue() will skin the object as a BigdataValue.
-
- - Move the flag bits into a high byte (in addition to the long
- termId). This will cluster URIs and Literals in their own parts
- of the lexicon. It will also increase the maximum #of URIs, plain
- or language code literals, XML literals, SIDs, etc. allowed in the
- lexicon significantly.
-
- - Add an inline bit? When inlining is enabled, some values will
- always be inlined (boolean, char, short, int, float, long, double,
- etc). However, an inline bit would let us differentiate this at
- runtime based solely on the VI state. This could be useful for
- inlining short plain text or language code literals.
-
- - Except for the inability to decode a Unicode sort key, we
- could then support a mode which inlined everything [we could
- do this anyway if we wanted to tolerate the redundancy of the
- data in the statement tuple value or if we restricted
- ourselves to ASCII].
-
- - Backward compatible mode where flags are in the low bits of the
- termId and we extract them to the flags byte in decode? Or just
- break with the past?
-
- - Inline blank nodes? (16 bytes).
-
- - Inlining all values for a datatype makes it possible to translate
- LT/GT style filters into OSP(C) key-range queries when casts are
- not used.
-
- - Inlining all numeric datatypes makes it possible to OSP(C) queries
- when casts are used. Otherwise, we can not translate a LT / GT
- style FILTERs into a key-range query.
-
- - Handle better coding of URIs (scheme, reversed dns components,
- path component compression)?
-
- - Join processing must handle IConstant<X> differently for term
- identifiers and inline literal values.
-
-SPOKeyOrder#encodeKey() : These methods should determine the #of key components based on isQuads()/getKeyArity()
-SPOKeyOrder#decodeKey()
-
-LexiconKeyBuilder - Review this class and then drop it. It was an
- alternative proposal for native datatype handling.
-
-ISPO#o()
-ISPO#get(int index)
-
-SPO#o()
-SPO(s,p,o)
-SPO(s,p,o,c)
-SPO(s,p,o,type)
-SPO(s,p,o,c,type)
-SPO(s,p,o) w/ IConstant<Long> arguments.
-SPO(IPredicate<ISPO>) handling of o.
-SPO(BigdataStatement) handling of o.
-SPO#hashCode() The o hash component is based on the decoded point in the value space?
-SPO#compareTo(ISPO) Must order the o within the value space and also put the value spaces into some order.
-SPO#equals(ISPO) Uses o == foo.o(), but must test
-
-ITermIdCodes - Must specify inline and long literal flag bits. Might
- also specify how the datatype code is represented.
- Update the javadoc since no longer the low bits of the
- term identifier. The TERM2ID index write procedure
- should continue to return 64-bit long integers since
- the caller can decorate them with the appropriate flags
- (is this also true for long literals which get written
- onto raw records?)
-
-ITermIndexCodes - This partitions the TERM2ID index using a one byte
- prefix. This probably does not need to be changed.
-
-KVOTermIdComparator - No change. This is only used for operations on
- the lexicon so it will only see term identifiers
- rather than inline values.
-
-LexiconRelation - RDF values which can be inlined should be handled
- differently. Also, we need to pass around
- collections of TermId or InternalValue objects, not
- Longs.
-
-TermIdEncoder - drop setFields() since the RDF Value type flags are
- now in a header byte.
-
- ? Modify to use the sign bit from the partition local
- counter and/or the partition identifier? Right now we
- do not roll those counters around to negative values,
- but doing so would allow us to have 4x as many term
- identifiers in the lexicon (2^32 vs 2^30).
-
-TermAdvancer - How do we form the successor for the TermAdvancer? Is
- this a fixed length bit string successor or do we
- decode the Statement and then get the successor() of
- the appropriate RDF Value (the s,p,o, or c).
-
-
-IRawTripleStore
-#addTerm(Value):long
-#getTerm(long id):Value - this is fine as far as it goes, but it is a special case now.
-#getTermId(Value):long - should probably return an InternalValue. Look at callers.
-#getAccessPath(...) - Redirect callers to SPORelation#getAccessPath()? Or just refactor to use InternalValue rather than long?
-#toString(long,...) - these use be changed to use InternalValues.
-
-AbstractTripleStore#
-isURI(), isLiteral(), isBNode(), isSID() - these static methods could
- be colocated on InternalValue with isTermId() etc.
-
-... Finish this survey of places where the code would have to change.
- Especially, locate all code which encodes or decodes the keys for
- the statement indices.
-
-BigdataStatement
-SPOTupleSerializer#statement2Key(s,p,o)
-
-Various SPO comparators.
-
-SPOPredicate
-
-...
-
-BaseAxioms
-
-========================================
-
-
-The current xsd:integer coding uses 2 bytes (less the signum bit) for
-the run length of the encoded BigInteger value. This is pragmatic
-since that would be a key component which is 32k long, and that is
-getting to the point where you do not want to be putting those things
-into the keys of the index. I will add some code to catch very large
-xsd:integer and xsd:decimal values and refuse to generate keys for
-them.
-
-I think that we should handle very long literals specially. Probably
-we should put them into raw journal records and store them under their
-MD5 hash code (that is 128 bits and collisions are exceedingly rare)
-in the TERM2ID index. This change would not effect the termIds as
-such, but rather than keys for the TERM2ID index (we would add a
-prefix code for very long literals there) and the values for the
-ID2TERM index (we would have a bit flag indicating that the value was
-a blog reference which needed to be dereferenced by reading on an
-appropriate record). This change could handle literals of up to
-several MB in size. Of course, this also presents a challenge for
-Literal#equals(Literal) for your clients.
-
-I have already mapped out support for this kind of blob reference for
-both standalone and scale-out, so there is nothing extra to do there.
-Just a little bit of work in how we write on and read from the TERM2ID
-and ID2TERM indices respectively. I am pretty sure that this change
-would not break either the code or the data.
-
-============================================================
-
-Note: appendUnsigned(byte v) already exists. However, the rest of
-these methods are new and ALL of them need unit tests. Also, consider
-passing in a data type with more bits to avoid problems with handling
-unsigned values in a signed data type.
-
- final public KeyBuilder appendUnsigned(final long v) {
-
- // performance tweak
- if (len + 8 > buf.length) ensureCapacity(len+8);
- // ensureFree(1);
-
- // big-endian.
- buf[len++] = (byte)(v >>> 56);
- buf[len++] = (byte)(v >>> 48);
- buf[len++] = (byte)(v >>> 40);
- buf[len++] = (byte)(v >>> 32);
- buf[len++] = (byte)(v >>> 24);
- buf[len++] = (byte)(v >>> 16);
- buf[len++] = (byte)(v >>> 8);
- buf[len++] = (byte)(v >>> 0);
-
- return this;
-
- }
-
- final public KeyBuilder appendUnsigned(final int v) {
-
- // performance tweak
- if (len + 4 > buf.length) ensureCapacity(len+4);
- // ensureFree(1);
-
- // big-endian
- buf[len++] = (byte)(v >>> 24);
- buf[len++] = (byte)(v >>> 16);
- buf[len++] = (byte)(v >>> 8);
- buf[len++] = (byte)(v >>> 0);
-
- return this;
-
- }
-
- final public KeyBuilder appendUnsigned(final short v) {
-
- // performance tweak
- if (len + 2 > buf.length) ensureCapacity(len+2);
- // ensureFree(1);
-
- // big-endian
- buf[len++] = (byte)(v >>> 8);
- buf[len++] = (byte)(v >>> 0);
-
- return this;
-
- }
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <btm...@us...> - 2010-07-23 22:22:02
|
Revision: 3283
http://bigdata.svn.sourceforge.net/bigdata/?rev=3283&view=rev
Author: btmurphy
Date: 2010-07-23 22:21:55 +0000 (Fri, 23 Jul 2010)
Log Message:
-----------
[trunk]: added non-graphical discovery tool and supporting classes (from dev-btm branch); to aid debugging discovery issues on a headless node where the graphical tools like the browser cannot be run
Modified Paths:
--------------
trunk/build.xml
Added Paths:
-----------
trunk/bigdata-jini/src/java/com/bigdata/attr/
trunk/bigdata-jini/src/java/com/bigdata/attr/ServiceInfo.java
trunk/bigdata-jini/src/java/com/bigdata/disco/
trunk/bigdata-jini/src/java/com/bigdata/disco/DiscoveryTool.java
trunk/bigdata-jini/src/java/com/bigdata/disco/config/
trunk/bigdata-jini/src/java/com/bigdata/disco/config/disco.config
trunk/bigdata-jini/src/java/com/bigdata/disco/config/logging.properties
trunk/bigdata-jini/src/java/com/bigdata/util/Util.java
trunk/bigdata-jini/src/java/com/bigdata/util/config/
trunk/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java
trunk/src/resources/bin/disco-tool
Added: trunk/bigdata-jini/src/java/com/bigdata/attr/ServiceInfo.java
===================================================================
--- trunk/bigdata-jini/src/java/com/bigdata/attr/ServiceInfo.java (rev 0)
+++ trunk/bigdata-jini/src/java/com/bigdata/attr/ServiceInfo.java 2010-07-23 22:21:55 UTC (rev 3283)
@@ -0,0 +1,218 @@
+/*
+
+Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+*/
+
+package com.bigdata.attr;
+
+import net.jini.entry.AbstractEntry;
+
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+public class ServiceInfo extends AbstractEntry {
+
+ private static final long serialVersionUID = 1L;
+
+ public ServiceInfo() { }
+
+ /**
+ * The unique id of the service associated with the given instance of
+ * this class.
+ *
+ * @serial
+ */
+ public UUID source;
+
+ /**
+ * <code>String</code> value that should contain a human readable name,
+ * description, or identifying string that can be used for display by
+ * clients managing or monitoring the associated service. Note that if
+ * this field is <code>null</code> or the empty string, then clients
+ * are encouraged to use the associated service's class name in such
+ * displays.
+ *
+ * @serial
+ */
+ public String serviceName;
+
+ /**
+ * <code>Map</code> whose key-value pairs consist of an
+ * <code>InetAddress</code> as key, and a <code>String</code> as
+ * corresponding value. Each key in the map represents an IP address
+ * associated with one of the network interface(s) of the node
+ * on which the associated service is exectuing. Each key's value
+ * is the name of the network interface to which that IP address
+ * is assigned.
+ *
+ * @serial
+ */
+ public Map<InetAddress,String> inetAddresses;
+
+ /**
+ * <code>String</code> value that is unique across all <i>nodes</i>
+ * in the system. With respect to the services executing on a given
+ * node, each such service should be able to discover, (or
+ * independently set through configuration for example), the information
+ * represented by the value of this field. For example, the value in this
+ * field might be set to the MAC address of an agreed upon network
+ * interface card (NIC) installed on the node on which the associated
+ * service is executing, or it might simply be that node's host name
+ * or IP address.
+ *
+ * @serial
+ */
+ public String nodeToken;
+
+ /**
+ * The unique id of the <i>node service</i> having the same
+ * <i>node token</i> as the token value referenced in this class;
+ * where a single node service executes on each node in the system,
+ * acting as a physical representation of the node on which this
+ * class' associated service also executes.
+ *
+ * @serial
+ */
+ public UUID nodeId;
+
+ /**
+ * <code>String</code> value representing the <i>logical name</i> that
+ * has been assigned to the <i>node</i> on which the associated service
+ * is executing.
+ *
+ * @serial
+ */
+ public String nodeName;
+
+ /**
+ * <code>Integer</code> value that uniquely identifies the
+ * physical position, within a given <i>rack</i>, in which the
+ * associated node has been mounted, relative to all other
+ * devices in the rack.
+ * <p>
+ * To understand the value that this field corresponds to, recall that
+ * the possible positions in which devices can be mounted in a given
+ * rack are generally identified by labeling the rack with a sequence
+ * of monotonically increasing integers in which the space in the rack
+ * from one number in the sequence to the next, represents <i>1U</i> in
+ * height; that is, 1 <i>rack unit</i> in height. As such, some devices
+ * will occupy only 1U of the rack, where the <i>U-number range</i> of
+ * such a device would correspond to the labels <b>n</b> and <b>n+1</b>;
+ * whereas other, larger devices may occupy 2U or 4U of the rack, and
+ * such devices would have U-number ranges of <b>n</b> to <b>n+2</b>,
+ * and <b>n</b> to <b>n+4</b> respectively. With respect to this
+ * so-called U-number range, the value of this field then, is the
+ * <i>first</i> number of the associated node's U-number range;
+ * that is, the vlaue of the U-number label adjacent to the top of
+ * the mounted node, relative to that device's position in the rack.
+ * For example, consider a node that is 4U in height and which
+ * is mounted in a given rack, occupying the positions ranging from
+ * U-number 75 to U-number 79. For that particular node, the value
+ * of this field would be 75; the first number of the range that is
+ * occupied.
+ *
+ * @serial
+ */
+ public Integer uNumber;
+
+ /**
+ * <code>String</code> value that uniquely identifies the <i>rack</i>
+ * (within a given <i>cage</i>) in the system that contains the
+ * <i>node</i> on which the associated service is executing.
+ *
+ * @serial
+ */
+ public String rack;
+
+ /**
+ * <code>String</code> value that uniquely identifies the <i>cage</i>
+ * (within a given <i>zone</i>) that contains the <i>rack</i> in which
+ * the associated service is executing.
+ *
+ * @serial
+ */
+ public String cage;
+
+ /**
+ * <code>String</code> value that uniquely identifies the <i>zone</i>
+ * (within a given <i>site</i>) that contains the <i>cage</i> in which
+ * the associated service is executing.
+ *
+ * @serial
+ */
+ public String zone;
+
+ /**
+ * <code>String</code> value that uniquely identifies the <i>site</i>
+ * (within a given <i>region</i>) that contains the <i>zone</i> in which
+ * the associated service is executing.
+ *
+ * @serial
+ */
+ public String site;
+
+ /**
+ * <code>String</code> value that uniquely identifies the <i>region</i>
+ * (within a given <i>geo</i>) that contains the <i>site</i> in which
+ * the associated service is executing.
+ *
+ * @serial
+ */
+ public String region;
+
+ /**
+ * <code>String</code> value that uniquely identifies a given <i>geo</i>
+ * designation, across <i>all</i> geo's, that contains the <i>region</i>
+ * in which the associated service is executing.
+ *
+ * @serial
+ */
+ public String geo;
+
+ /**
+ * @see <code>net.jini.entry.AbstractEntry#equals</code>.
+ */
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj);
+ }
+
+ /**
+ * @see <code>net.jini.entry.AbstractEntry#hashCode</code>.
+ */
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ /**
+ * @see <code>net.jini.entry.AbstractEntry#toString</code>.
+ */
+ @Override
+ public String toString() {
+ return super.toString();
+ }
+}
Property changes on: trunk/bigdata-jini/src/java/com/bigdata/attr/ServiceInfo.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: trunk/bigdata-jini/src/java/com/bigdata/disco/DiscoveryTool.java
===================================================================
--- trunk/bigdata-jini/src/java/com/bigdata/disco/DiscoveryTool.java (rev 0)
+++ trunk/bigdata-jini/src/java/com/bigdata/disco/DiscoveryTool.java 2010-07-23 22:21:55 UTC (rev 3283)
@@ -0,0 +1,375 @@
+/*
+
+Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+*/
+package com.bigdata.disco;
+
+import com.bigdata.util.Util;
+
+import com.sun.jini.config.Config;
+import net.jini.config.Configuration;
+import net.jini.config.ConfigurationException;
+import net.jini.config.ConfigurationProvider;
+import net.jini.core.constraint.MethodConstraints;
+import net.jini.core.discovery.LookupLocator;
+import net.jini.core.entry.Entry;
+import net.jini.core.lookup.ServiceItem;
+import net.jini.core.lookup.ServiceTemplate;
+import net.jini.discovery.ConstrainableLookupLocator;
+import net.jini.discovery.DiscoveryManagement;
+import net.jini.discovery.DiscoveryGroupManagement;
+import net.jini.discovery.DiscoveryLocatorManagement;
+import net.jini.lookup.LookupCache;
+import net.jini.lookup.ServiceDiscoveryEvent;
+import net.jini.lookup.ServiceDiscoveryListener;
+import net.jini.lookup.ServiceDiscoveryManager;
+import net.jini.lookup.ServiceItemFilter;
+
+import static java.lang.Boolean.parseBoolean;
+import static java.lang.Integer.parseInt;
+import static java.lang.Long.parseLong;
+import static java.lang.Thread.sleep;
+import static java.util.Arrays.asList;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.synchronizedMap;
+import static net.jini.discovery.Constants.discoveryPort;
+import static net.jini.discovery.DiscoveryGroupManagement.ALL_GROUPS;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.text.DateFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.WeakHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class DiscoveryTool {
+
+ private static final String COMPONENT_NAME = DiscoveryTool.class.getName();
+ private static final String CONFIG_ARG = "config";
+ private static final String GROUP_ARG_PREFIX = "group";
+ private static final String LOCATOR_ARG_PREFIX = "locator";
+ private static final String SERVICE_ARG_PREFIX = "service";
+ private static final String ALL_GROUPS_ARG = "allGroups";
+ private static final String TIMEOUT_ARG = "timeout";
+ private static final String OUTPUT_ARG = "output";
+ private static final String VERBOSE_ARG = "verbose";
+ private static final Pattern hostPortPattern =
+ Pattern.compile("(.+):(\\d+)");
+
+ public static void main(String[] args) throws Exception {
+ new DiscoveryTool(args).run();
+ }
+
+ private final Map<Class, Collection<Class>> typeCache =
+ synchronizedMap(new WeakHashMap<Class, Collection<Class>>());
+ private final Configuration config;
+ private final Collection<String> groups = new ArrayList<String>();
+ private final Collection<LookupLocator> locators =
+ new ArrayList<LookupLocator>();
+ private final Collection<Pattern> serviceTypePatterns =
+ new ArrayList<Pattern>();
+ private final boolean allGroups;
+ private final Long timeout;
+ private final PrintStream out;
+ private final boolean verbose;
+ private final DateFormat dateFormat;
+ private final Set<String> uninterestingInterfaces;
+
+ private DiscoveryTool(String[] rawArgs)
+ throws ConfigurationException, IOException
+ {
+ Arguments args = new Arguments(rawArgs);
+
+ config = ConfigurationProvider.getInstance
+ ( new String[]{ args.get(CONFIG_ARG) },
+ (this.getClass()).getClassLoader() );
+
+ for (int i = 0; ; i++) {
+ String argName = GROUP_ARG_PREFIX + i;
+ if (!args.contains(argName)) {
+ break;
+ }
+ groups.add(args.get(argName));
+ }
+
+ MethodConstraints locatorConstraints =
+ (MethodConstraints)config.getEntry(COMPONENT_NAME,
+ "lookupLocatorConstraints",
+ MethodConstraints.class,
+ null);
+ for (int i = 0; ; i++) {
+ String argName = LOCATOR_ARG_PREFIX + i;
+ if (!args.contains(argName)) {
+ break;
+ }
+ String val = args.get(argName);
+ Matcher m = hostPortPattern.matcher(val);
+ locators.add(m.matches() ?
+ new ConstrainableLookupLocator(
+ m.group(1), parseInt(m.group(2)), locatorConstraints) :
+ new ConstrainableLookupLocator(
+ val, discoveryPort, locatorConstraints));
+ }
+
+ for (int i = 0; ; i++) {
+ String argName = SERVICE_ARG_PREFIX + i;
+ if (!args.contains(argName)) {
+ break;
+ }
+ serviceTypePatterns.add(Pattern.compile(args.get(argName)));
+ }
+
+ timeout = args.contains(TIMEOUT_ARG) ?
+ parseLong(args.get(TIMEOUT_ARG)) : null;
+
+ if (args.contains(OUTPUT_ARG)) {
+ String val = args.get(OUTPUT_ARG);
+ out = val.equals("-") ? System.out : new PrintStream(val);
+ } else {
+ out = System.out;
+ }
+
+ allGroups = parseBoolean(args.get(ALL_GROUPS_ARG));
+ verbose = parseBoolean(args.get(VERBOSE_ARG));
+
+ dateFormat = (DateFormat)Config.getNonNullEntry
+ (config, COMPONENT_NAME, "dateFormat",
+ DateFormat.class, DateFormat.getInstance());
+
+ uninterestingInterfaces =
+ new HashSet<String>(asList((String[])Config.getNonNullEntry
+ (config, COMPONENT_NAME,
+ "uninterestingInterfaces",
+ String[].class,
+ new String[0])));
+ }
+
+ void run()
+ throws ConfigurationException, IOException, InterruptedException
+ {
+ DiscoveryManagement discoveryManager = null;
+ ServiceDiscoveryManager serviceDiscovery = null;
+
+ try {
+ println(getDateString(), ": starting discovery");
+
+ discoveryManager =
+ Util.getDiscoveryManager(config, COMPONENT_NAME);
+
+ verbose("created lookup discovery manager: ", discoveryManager);
+
+ verbose("groups: ", allGroups ? "<all>" : groups.toString());
+ ((DiscoveryGroupManagement) discoveryManager).setGroups(
+ allGroups ?
+ ALL_GROUPS : groups.toArray(new String[groups.size()]));
+ verbose("locators: ", locators);
+ ((DiscoveryLocatorManagement) discoveryManager).setLocators(
+ locators.toArray(new LookupLocator[locators.size()]));
+
+ serviceDiscovery =
+ new ServiceDiscoveryManager(discoveryManager, null, config);
+ verbose("created service discovery manager: ", serviceDiscovery);
+
+ verbose("service type patterns: ", serviceTypePatterns);
+ ServiceItemFilter serviceFilter = serviceTypePatterns.isEmpty() ?
+ null : new ServiceTypeFilter(serviceTypePatterns);
+ LookupCache cache = serviceDiscovery.createLookupCache(
+ null, serviceFilter, new Listener());
+ verbose("created lookup cache: ", cache);
+
+ verbose(
+ "timeout: ", (timeout != null) ? (timeout + " ms") : "<none>");
+ if (timeout != null) {
+ sleep(timeout);
+ } else {
+ while (true) {
+ sleep(Long.MAX_VALUE);
+ }
+ }
+
+ verbose("shutting down");
+
+ } finally {
+ if (serviceDiscovery != null) {
+ serviceDiscovery.terminate();
+ }
+ if (discoveryManager != null) {
+ discoveryManager.terminate();
+ }
+ }
+ }
+
+ void println(Object... args) {
+ StringBuilder sb = new StringBuilder();
+ for (Object arg : args) {
+ sb.append(arg);
+ }
+ out.println(sb);
+ }
+
+ void verbose(Object... args) {
+ if (verbose) {
+ println(args);
+ }
+ }
+
+ String getDateString() {
+ return dateFormat.format(new Date());
+ }
+
+ Collection<Class> getTypes(Object obj) {
+ if (obj == null) {
+ return emptyList();
+ }
+ Class cl = obj.getClass();
+ Collection<Class> types = typeCache.get(cl);
+ if (types == null) {
+ Set<Class> s = new HashSet<Class>();
+ accumulateTypes(cl, s);
+ types = new ArrayList<Class>(s);
+ typeCache.put(cl, types);
+ }
+ return types;
+ }
+
+ private static void accumulateTypes(Class cl, Set<Class> types) {
+ if (cl != null && types.add(cl)) {
+ accumulateTypes(cl.getSuperclass(), types);
+ for (Class intf : cl.getInterfaces()) {
+ accumulateTypes(intf, types);
+ }
+ }
+ }
+
+ private static String join(Iterable<?> i, String delim) {
+ Iterator<?> iter = i.iterator();
+ if (!iter.hasNext()) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ sb.append(iter.next());
+ while (iter.hasNext()) {
+ sb.append(delim);
+ sb.append(iter.next());
+ }
+ return sb.toString();
+ }
+
+ private class Listener implements ServiceDiscoveryListener {
+
+ Listener() {
+ }
+
+ public synchronized void serviceAdded(ServiceDiscoveryEvent event) {
+ println(getDateString(), ": service added");
+ printServiceItem(event.getPostEventServiceItem());
+ }
+
+ public synchronized void serviceChanged(ServiceDiscoveryEvent event) {
+ println(getDateString(), ": service changed");
+ printServiceItem(event.getPostEventServiceItem());
+ }
+
+ public synchronized void serviceRemoved(ServiceDiscoveryEvent event) {
+ println(getDateString(), ": service removed");
+ printServiceItem(event.getPreEventServiceItem());
+ }
+
+ private void printServiceItem(ServiceItem item) {
+ Collection<String> typeNames = new ArrayList<String>();
+ for (Class type : getTypes(item.service)) {
+ if (type.isInterface() &&
+ !uninterestingInterfaces.contains(type.getName()))
+ {
+ typeNames.add(type.getName());
+ }
+ }
+
+ println(" Service ID: ", item.serviceID);
+ println(" Types: ", join(typeNames, ", "));
+ println(" Attributes:");
+ for (Entry e : item.attributeSets) {
+ println(" ", e);
+ }
+ println();
+ }
+ }
+
+ private class ServiceTypeFilter implements ServiceItemFilter {
+
+ private final Collection<Pattern> serviceTypePatterns;
+
+ ServiceTypeFilter(Collection<Pattern> serviceTypePatterns) {
+ this.serviceTypePatterns = serviceTypePatterns;
+ }
+
+ public boolean check(ServiceItem item) {
+ for (Class type : getTypes(item.service)) {
+ for (Pattern pattern : serviceTypePatterns) {
+ if (pattern.matcher(type.getName()).find()) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+ }
+
+ private static class Arguments {
+
+ private static final Pattern pattern = Pattern.compile("(.+)=(.+)");
+ private final Map<String, String> map = new HashMap<String, String>();
+
+ Arguments(String[] rawArgs) {
+ for (String arg : rawArgs) {
+ Matcher m = pattern.matcher(arg);
+ if (!m.matches()) {
+ throw new IllegalArgumentException(
+ "invalid argument: " + arg);
+ }
+ map.put(m.group(1), m.group(2));
+ }
+ }
+
+ boolean contains(String name) {
+ return map.containsKey(name);
+ }
+
+ String get(String name) {
+ String val = map.get(name);
+ if (val != null) {
+ return val;
+ } else {
+ throw new IllegalArgumentException("no value for " + name);
+ }
+ }
+ }
+}
Property changes on: trunk/bigdata-jini/src/java/com/bigdata/disco/DiscoveryTool.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: trunk/bigdata-jini/src/java/com/bigdata/disco/config/disco.config
===================================================================
--- trunk/bigdata-jini/src/java/com/bigdata/disco/config/disco.config (rev 0)
+++ trunk/bigdata-jini/src/java/com/bigdata/disco/config/disco.config 2010-07-23 22:21:55 UTC (rev 3283)
@@ -0,0 +1,65 @@
+/* Configuration file for the discovery tool */
+
+// Note that this file has no tunable parameters
+// and should not be modified.
+
+import java.text.DateFormat;
+
+import net.jini.constraint.BasicMethodConstraints;
+import net.jini.core.constraint.ConnectionRelativeTime;
+import net.jini.core.constraint.InvocationConstraints;
+import net.jini.core.discovery.LookupLocator;
+import net.jini.discovery.LookupDiscoveryManager;
+import net.jini.jeri.BasicILFactory;
+import net.jini.jeri.BasicJeriExporter;
+import net.jini.jeri.tcp.TcpServerEndpoint;
+
+import com.bigdata.util.config.NicUtil;
+
+com.bigdata.disco.DiscoveryTool {
+
+ private static exportIpAddr =
+ NicUtil.getIpAddress("${exportNic}", 0, "${exportHost}");
+
+ lookupLocatorConstraints = null;
+
+ dateFormat =
+ DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.FULL);
+
+ uninterestingInterfaces =
+ new String[] { "java.io.Serializable",
+ "java.rmi.Remote",
+ "net.jini.admin.Administrable",
+ "net.jini.core.constraint.RemoteMethodControl",
+ "net.jini.id.ReferentUuid",
+ "com.bigdata.service.Service",
+ "com.bigdata.service.EventReceivingService"
+ };
+
+
+ static discoveryManager =
+ new LookupDiscoveryManager
+ (new String[]{}, new LookupLocator[]{}, null, this);
+}
+
+net.jini.discovery.LookupDiscovery {
+
+ multicastRequestHost =
+ com.bigdata.disco.DiscoveryTool.exportIpAddr;
+
+ multicastInterfaces =
+ NicUtil.getNetworkInterfaceArray("${networkInterface}");
+}
+
+net.jini.lookup.ServiceDiscoveryManager {
+
+ private static serverILFactory =
+ new BasicILFactory(
+ new BasicMethodConstraints(
+ new InvocationConstraints(
+ new ConnectionRelativeTime(10000L), null)),
+ null);
+ eventListenerExporter =
+ new BasicJeriExporter(TcpServerEndpoint.getInstance(com.bigdata.disco.DiscoveryTool.exportIpAddr, 0),
+ serverILFactory, false, false);
+}
Property changes on: trunk/bigdata-jini/src/java/com/bigdata/disco/config/disco.config
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: trunk/bigdata-jini/src/java/com/bigdata/disco/config/logging.properties
===================================================================
--- trunk/bigdata-jini/src/java/com/bigdata/disco/config/logging.properties (rev 0)
+++ trunk/bigdata-jini/src/java/com/bigdata/disco/config/logging.properties 2010-07-23 22:21:55 UTC (rev 3283)
@@ -0,0 +1,25 @@
+######################################################################
+# Log4j configuration file for the discovery tool
+######################################################################
+
+# FATAL
+# ERROR
+# WARN
+# INFO
+# DEBUG
+# TRACE
+
+# All messages are directed to stderr.
+
+# log4j setup
+log4j.rootLogger=WARN, Console
+log4j.appender.Console=org.apache.log4j.ConsoleAppender
+log4j.appender.Console.target=System.err
+log4j.appender.Console.layout=org.apache.log4j.PatternLayout
+log4j.appender.Console.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
+
+# java.util.logging setup
+.level=WARNING
+handlers=com.bigdata.util.config.Log4jLoggingHandler
+com.sun.jini.logging.interval=60000
+
Property changes on: trunk/bigdata-jini/src/java/com/bigdata/disco/config/logging.properties
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: trunk/bigdata-jini/src/java/com/bigdata/util/Util.java
===================================================================
--- trunk/bigdata-jini/src/java/com/bigdata/util/Util.java (rev 0)
+++ trunk/bigdata-jini/src/java/com/bigdata/util/Util.java 2010-07-23 22:21:55 UTC (rev 3283)
@@ -0,0 +1,426 @@
+/*
+
+Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+*/
+
+package com.bigdata.util;
+
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.LogUtil;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import com.sun.jini.config.Config;
+import com.sun.jini.thread.InterruptedStatusThread;
+import net.jini.config.Configuration;
+import net.jini.config.ConfigurationException;
+import net.jini.config.NoSuchEntryException;
+import net.jini.core.lookup.ServiceID;
+import net.jini.discovery.DiscoveryManagement;
+import net.jini.discovery.DiscoveryGroupManagement;
+import net.jini.discovery.DiscoveryLocatorManagement;
+import net.jini.discovery.LookupDiscoveryManager;
+import net.jini.export.Exporter;
+import net.jini.lookup.JoinManager;
+import net.jini.lookup.ServiceDiscoveryEvent;
+import net.jini.lookup.ServiceDiscoveryManager;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.UUID;
+
+/**
+ * Miscellaneous, convenient utility methods.
+ */
+public class Util {
+
+ public static <T extends Comparable<T>> T max(final T... elements){
+ T max = elements[0];
+ for (final T element : elements) {
+ if(0 < element.compareTo(max)) {
+ max = element;
+ }
+ }
+ return max;
+ }
+
+ public static <T extends Comparable<T>> T min(final T... elements){
+ T min = elements[0];
+ for (final T element : elements) {
+ if(0 > element.compareTo(min)) {
+ min = element;
+ }
+ }
+ return min;
+ }
+
+ /* Convenience method that can be called when a service exits, or
+ * when failure occurs during the service's initialization process.
+ * This method un-does any work that may have already been completed;
+ * for example, un-exports the service if it has already been
+ * exported, closes any open sockets or file descriptors, terminates
+ * threads that may have been started, etc.
+ * <p>
+ * Note that multiple versions of this method are provided. One version
+ * is intended to be used by entities that act only as a service (that
+ * is, entities that export a proxy and use a <code>JoinManager</code>).
+ * One version is intended to be used by entities that act only as a client
+ * (that is, entites that use a <code>ServiceDiscoveryManager</code>).
+ * And the final version can be used by entities that act as both a
+ * service and as a client.
+ */
+ public static void cleanupOnExit
+ (Object innerProxy,
+ Exporter serverExporter,
+ JoinManager joinManager,
+ DiscoveryManagement discoveryManager)
+ {
+ cleanupOnExit(innerProxy, serverExporter, joinManager,
+ null, discoveryManager);
+
+ }
+
+ public static void cleanupOnExit
+ (ServiceDiscoveryManager serviceDiscoveryManager,
+ DiscoveryManagement discoveryManager)
+ {
+ cleanupOnExit(null, null, null,
+ serviceDiscoveryManager, discoveryManager);
+ }
+
+
+ public static void cleanupOnExit
+ (Object innerProxy,
+ Exporter serverExporter,
+ JoinManager joinManager,
+ ServiceDiscoveryManager serviceDiscoveryManager,
+ DiscoveryManagement discoveryManager)
+ {
+ if(innerProxy != null) {
+ try {
+ if(serverExporter != null) serverExporter.unexport(true);
+ } catch(Throwable t) { }
+ }
+
+ if(joinManager != null) {
+ try {
+ joinManager.terminate();
+ } catch(Throwable t) { }
+ }
+
+ if(serviceDiscoveryManager != null) {
+ try {
+ serviceDiscoveryManager.terminate();
+ } catch(Throwable t) { }
+ }
+
+ if(discoveryManager != null) {
+ try {
+ discoveryManager.terminate();
+ } catch(Throwable t) { }
+ }
+ }
+
+
+ /**
+ * Unexports the remote object that was exported by the given
+ * <code>Exporter</code> parameter; which removes the object
+ * from the RMI runtime so that the object can no longer accept
+ * incoming remote calls.er accept incoming RMI calls.
+ * <P>
+ * This method first makes an attempt to unexport the object
+ * 'gracefully'. That is, for a finite period of time, an attempt
+ * is made to allow all calls to the object that are in progress,
+ * or pending, to complete before the object is actually unexported.
+ * If, after that finite period of time, the object has not been
+ * successfully unexported, the object is then 'forcibly' unexported;
+ * that is, the object is unexported even if there are calls to
+ * the object that are in progress or still pending.
+ * <P>
+ * Upon successfully unexporting the given <code>Exporter</code>,
+ * <code>true</code> is returned. If the given <code>Exporter</code>
+ * cannot be unexported, or if the value input for that parameter
+ * is <code>null</code> or has not exported any interfaces, then
+ * <code>false</code> is returned.
+ */
+ public static boolean unexportRemoteObject(Exporter exporter) {
+ if (exporter == null) return false;
+
+ // delay no more than 1 minute
+ final long endTime = System.currentTimeMillis() + (1L*60L*1000L);
+ boolean unexported = false;
+ try {
+ // Unexport only if there are no pending or in-progress calls
+ while (!unexported && System.currentTimeMillis() < endTime) {
+ unexported = exporter.unexport(false);//do not force
+ if (!unexported) Thread.yield();
+ }//end loop
+ if (!unexported) unexported = exporter.unexport(true);//force
+ } catch ( IllegalStateException e ) {
+ // Thrown if no object has been exported with the
+ // Exporter instance
+ return false;
+ }
+ return unexported;
+ }
+
+
+ /**
+ * Convenience method that can be called in an entity's constructor
+ * when failure occurs during the initialization process. This
+ * method simply rethrows the given <code>Throwable</code> so the
+ * constructor doesn't have to.
+ */
+ public static void handleInitThrowable(Throwable t, Logger logger)
+ throws IOException,
+ ConfigurationException
+ {
+ if( logger != null ) {
+ logger.log(Level.FATAL, "initialization failure ... ", t);
+ } else {
+ System.err.println("FATAL: initialization failure ... "+t);
+ }//endif
+ if (t instanceof IOException) {
+ throw (IOException)t;
+ } else if (t instanceof ConfigurationException) {
+ throw (ConfigurationException)t;
+ } else if (t instanceof RuntimeException) {
+ throw (RuntimeException)t;
+ } else if (t instanceof Error) {
+ throw (Error)t;
+ }//endif
+ }
+
+ /**
+ * Convenience method that returns a <code>String</code> containing
+ * a common-separated list the elements (group names) of the given
+ * array.
+ */
+ public static String writeGroupArrayToString(String[] groups) {
+ if(groups == null) {
+ return new String("[ALL_GROUPS]");
+ }//endif
+ if(groups.length <= 0) {
+ return new String("[]");
+ }//endif
+ StringBuffer strBuf = null;
+ if(groups[0].compareTo("") == 0) {
+ strBuf = new StringBuffer("[The PUBLIC Group");
+ } else {
+ strBuf = new StringBuffer("["+groups[0]);
+ }//endif
+ for(int i=1;i<groups.length;i++) {
+ if(groups[i].compareTo("") == 0) {
+ strBuf.append(", The PUBLIC Group");
+ } else {
+ strBuf.append(", ").append(groups[i]);
+ }//endif
+ }//end loop
+ strBuf.append("]");
+ return strBuf.toString();
+ }
+
+ /**
+ * Convenience method that returns a <code>String</code> containing
+ * a common-separated list the elements (locators) of the given
+ * array.
+ */
+ public static String writeArrayElementsToString(Object[] arr) {
+ if(arr == null) return new String("[]");
+ if(arr.length <= 0) {
+ return new String("[]");
+ }//endif
+ StringBuffer strBuf = new StringBuffer("["+arr[0]);
+ for(int i=1;i<arr.length;i++){
+ strBuf.append(", ").append(arr[i]);
+ }//end loop
+ strBuf.append("]");
+ return strBuf.toString();
+ }
+
+ /**
+ * Convenience method to simplify the throwing of exceptions with embedded
+ * causes (avoids having to cast the return value of Throwable.initCause
+ * back to the exception's type). Use as follows:
+ * <pre>
+ * throw Util.initCause(new SomeException("foo"), cause);
+ * </pre>
+ */
+ public static <T extends Throwable> T initCause(T t, Throwable cause) {
+ t.initCause(cause);
+ return t;
+ }
+
+ /**
+ * Verifies that all non-<code>null</code> elements of the given
+ * <code>Collection</code> are assignable to the specified type,
+ * throwing a <code>ClassCastException</code> if any are not.
+ */
+ public static void checkElementTypes(Collection<?> c, Class<?> type) {
+ for (Object elt : c) {
+ if (!type.isInstance(elt)) {
+ throw new ClassCastException(
+ elt + " not assignable to " + type);
+ }
+ }
+ }
+
+ /**
+ * Returns a UUID with the same bit value as the given
+ * <code>ServiceID</code>.
+ */
+ public static UUID toUUID(ServiceID serviceId) {
+ return new UUID( serviceId.getMostSignificantBits(),
+ serviceId.getLeastSignificantBits() );
+ }
+
+ /**
+ * Returns a string representation of the given
+ * <code>ServiceDiscoveryEvent</code> (since
+ * <code>ServiceDiscoveryEvent</code> doesn't define
+ * its own <code>toString</code> method).
+ */
+ public static String eventToString(ServiceDiscoveryEvent event) {
+ return "ServiceDiscoveryEvent[source=" + event.getSource() +
+ ",preEventItem=" + event.getPreEventServiceItem() +
+ ",postEventItem=" + event.getPostEventServiceItem() + "]";
+ }
+
+ /**
+ * Convenience method that encapsulates common functions that services
+ * or clients may wish to perform to be able to discover lookup services
+ * in the system.
+ * <p>
+ * This method retrieves and returns a lookup discovery manager from
+ * the given <code>Configuration</code>. If no lookup discovery manager
+ * has been configured, this method will return an instance of the
+ * <code>LookupDiscoveryManager</code> helper utility class,
+ * initialized to discover NO_GROUPS and no locators. When such a
+ * discovery manager is returned, the calling entity can call the
+ * <code>setGroups</code> and/or </code>setLocators</code> method
+ * to initiate the lookup discovery process.
+ * <p>
+ * Note that this method expects that the discovery manager
+ * that has been configured is an instance of both
+ * <code>DiscoveryGroupManagement</code> and
+ * <code>DiscoveryLocatorManagement</code>.
+ *
+ * @param config The calling service's <code>Configuration</code>
+ * from which this method will retrieve the items
+ * needed to perform the desired initialization.
+ *
+ * @param componentName <code>String</code> whose value is the name of
+ * the <i>component</i> used to index the calling
+ * service's configuration <i>entries</i>.
+ *
+ * @param entryName <code>String</code> whose value is the name of
+ * the configuration entry that references the
+ * the desired lookup discovery manager instance
+ * specified in the configuration.
+ *
+ * @return An instance of <code>DiscoveryManagement</code> that supports
+ * both group and locator discovery; where the instance returned
+ * is either retrieved from the given <code>Configuration</code>,
+ * or is a default instance of <code>LookupDiscoveryManager</code>.
+ *
+ * @throws <code>ConfigurationException</code> when there is a problem
+ * retrieving the desired entry from the configuration.
+ *
+ * @throws IOException when there is a problem with multicast discovery.
+ */
+ public static DiscoveryManagement getDiscoveryManager
+ (Configuration config,
+ String componentName,
+ String entryName )
+ throws ConfigurationException,
+ IOException
+ {
+ // The discovery manager must be an instance of both
+ // DiscoveryGroupManagement and DiscoveryLocatorManagement, so that
+ // the groupsToJoin and locatorsToJoin can both be retrieved from
+ //the discovery manager and displayed.
+ DiscoveryManagement dMgr;
+ try {
+ dMgr = (DiscoveryManagement)Config.getNonNullEntry
+ (config,
+ componentName,
+ entryName,
+ DiscoveryManagement.class);
+ if( !(dMgr instanceof DiscoveryGroupManagement) ) {
+ throw new ConfigurationException
+ (entryName + " entry must "
+ +"implement DiscoveryGroupManagment");
+ }
+ if( !(dMgr instanceof DiscoveryLocatorManagement) ) {
+ throw new ConfigurationException
+ (entryName + " entry must "
+ +"implement DiscoveryLocatorManagement");
+ }
+ } catch (NoSuchEntryException e) {
+ return ( new LookupDiscoveryManager
+ (ConfigDeployUtil.getGroupsToDiscover(),
+ ConfigDeployUtil.getLocatorsToDiscover(),
+ null, config) );
+ }
+ return dMgr;
+ }
+
+ /**
+ * Retrieves and returns a lookup discovery manager from the given
+ * <code>Configuration</code>, using a default entry name of
+ * <i>discoveryManager</i>.
+ */
+ public static DiscoveryManagement getDiscoveryManager
+ (Configuration config,
+ String componentName)
+ throws ConfigurationException,
+ IOException
+ {
+ return getDiscoveryManager(config, componentName, "discoveryManager");
+ }
+
+ public static class WaitOnInterruptThread extends InterruptedStatusThread {
+ private Logger logger;
+ public WaitOnInterruptThread(final Logger logger) {
+ super("WaitOnInterruptThread");
+ setDaemon(true);
+ this.logger = (logger == null ?
+ LogUtil.getLog4jLogger((this.getClass()).getName()) :
+ logger);
+ }
+ public void run() {
+ while (!hasBeenInterrupted()) {
+ try {
+ Thread.sleep(Long.MAX_VALUE);
+ } catch (InterruptedException e) {
+ if( logger.isDebugEnabled() ) {
+ logger.log(Level.DEBUG,
+ "Util.WaitOnInterruptThread: "
+ +"interrupt received");
+ }
+ }
+ }
+ }
+ }
+}
Property changes on: trunk/bigdata-jini/src/java/com/bigdata/util/Util.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: trunk/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java
===================================================================
--- trunk/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java (rev 0)
+++ trunk/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java 2010-07-23 22:21:55 UTC (rev 3283)
@@ -0,0 +1,595 @@
+/*
+
+Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+*/
+
+package com.bigdata.util.config;
+
+import com.bigdata.attr.ServiceInfo;
+
+import com.sun.jini.config.Config;
+import net.jini.config.Configuration;
+import net.jini.config.ConfigurationException;
+import net.jini.core.discovery.LookupLocator;
+import net.jini.discovery.DiscoveryManagement;
+import net.jini.discovery.DiscoveryGroupManagement;
+import net.jini.discovery.DiscoveryLocatorManagement;
+import net.jini.discovery.LookupDiscoveryManager;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.SocketException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.UUID;
+
+/**
+ * Utility class containing a number of convenient methods that encapsulate
+ * common functions related to configuration and deployment of the entity.
+ * The methods of this class are <code>static</code> so that they can be
+ * invoked from within a Jini configuration.
+ */
+public class ConfigDeployUtil {
+
+ private static Properties deploymentProps = null;
+
+ private static final String DEFAULT = ".default";
+ private static final String STRINGVALS = ".stringvals";
+ private static final String MAX = ".max";
+ private static final String MIN = ".min";
+ private static final String DESCRIPTION = ".description";
+ private static final String TYPE = ".type";
+
+ public static String getString(String parameter)
+ throws ConfigurationException
+ {
+ String value = get(parameter);
+ validateString(parameter, value);
+ return value;
+ }
+
+ public static String[] getStringArray(String parameter)
+ throws ConfigurationException
+ {
+ String[] value;
+ value = validateStringArray(parameter, get(parameter));
+ return value;
+ }
+
+ public static int getInt(String parameter) throws ConfigurationException {
+ int value;
+ value = validateInt(parameter, get(parameter));
+ return value;
+ }
+
+ public static long getLong(String parameter) throws ConfigurationException
+ {
+ long value;
+ value = validateLong(parameter, get(parameter));
+ return value;
+ }
+
+ public static boolean getBoolean(String parameter)
+ throws ConfigurationException {
+ boolean boolValue = false;
+ String value;
+ try {
+ value = get(parameter);
+ } catch (Exception ex) {
+ throw new ConfigurationException("parameter value ["+parameter+"] "
+ +"neither 'true' nor 'false'");
+ }
+ if( value != null && value.equalsIgnoreCase("true")
+ || value.equalsIgnoreCase("false") )
+ {
+ boolValue = Boolean.parseBoolean(value);
+ } else {
+ throw new ConfigurationException("parameter value ["+parameter+"] "
+ +"neither 'true' nor 'false'");
+ }
+ return boolValue;
+ }
+
+ public static String getDescription(String parameter)
+ throws ConfigurationException
+ {
+ String value;
+ if(deploymentProps == null) {
+ deploymentProps = new Properties();
+ loadDeployProps(deploymentProps);
+ }
+ value = deploymentProps.getProperty(parameter + DESCRIPTION);
+ return value;
+ }
+
+ public static String getType(String parameter)
+ throws ConfigurationException
+ {
+ String value;
+ if (deploymentProps == null) {
+ deploymentProps = new Properties();
+ loadDeployProps(deploymentProps);
+ }
+ value = deploymentProps.getProperty(parameter + TYPE);
+ return value;
+ }
+
+ public static String getDefault(String parameter)
+ throws ConfigurationException
+ {
+ String value;
+ if (deploymentProps == null) {
+ deploymentProps = new Properties();
+ loadDeployProps(deploymentProps);
+ }
+ value = deploymentProps.getProperty(parameter + DEFAULT);
+ if (value == null) {
+ throw new ConfigurationException
+ ("deployment parameter not found ["+parameter+"]");
+ }
+ return value;
+ }
+
+ /**
+ * Returns a <code>String</code> array whose elments represent the
+ * lookup service groups to discover. If the system property named
+ * "federation.name" is set then that value be used; otherwise,
+ * the deployment properties files will be consulted.
+ *
+ * @throws ConfigurationException if the groups cannot be determined.
+ */
+ public static String[] getGroupsToDiscover() throws ConfigurationException
+ {
+ String fedNameStr = System.getProperty("federation.name");
+ if(fedNameStr == null) {
+ fedNameStr = getString("federation.name");
+ }
+ return fedNameStr.split(",");
+ }
+
+ /**
+ * Returns an array of <code>LookupLocator</code> instances that can
+ * each be used to discover a specific lookup service.
+ *
+ * @throws ConfigurationException if the locators cannot be determined.
+ */
+ public static LookupLocator[] getLocatorsToDiscover()
+ throws ConfigurationException
+ {
+ return new LookupLocator[]{};
+ }
+
+ /**
+ * Returns an instance of the <code>ServiceInfo</code> attribute class,
+ * initialized to the values specified in the deployment properties
+ * files.
+ */
+ public static ServiceInfo initServiceInfo(UUID source, String serviceName)
+ throws SocketException, ConfigurationException
+ {
+ ServiceInfo serviceInfo = new ServiceInfo();
+ serviceInfo.source = source;
+ serviceInfo.serviceName = serviceName;
+
+ serviceInfo.inetAddresses = NicUtil.getInetAddressMap();
+
+ // Get the common token that all services running on the same
+ // node agree on. Use the MAC address or IP address as default token
+ String nodeNicName = getString("node.serviceNetwork");
+ String nodeIp = NicUtil.getIpAddress(nodeNicName, false);
+ serviceInfo.nodeToken = NicUtil.getMacAddress(nodeNicName);
+ if(serviceInfo.nodeToken == null) serviceInfo.nodeToken = nodeIp;
+
+ serviceInfo.nodeId = null;//not set until a node service exists
+ serviceInfo.nodeName = getString("node.name");
+
+ serviceInfo.uNumber = getInt("node.uNumber");
+
+ serviceInfo.rack = getString("node.rack");
+ serviceInfo.cage = getString("node.cage");
+ serviceInfo.zone = getString("node.zone");
+ serviceInfo.site = getString("node.site");
+ serviceInfo.region = getString("node.region");
+ serviceInfo.geo = getString("node.geo");
+
+ return serviceInfo;
+ }
+
+
+ private static String get(String parameter) throws ConfigurationException {
+ String value;
+ if (deploymentProps == null) {
+ deploymentProps = new Properties();
+ loadDeployProps(deploymentProps);
+ }
+ value = deploymentProps.getProperty(parameter);
+ if (value == null) value = getDefault(parameter);
+ return value;
+ }
+
+ private static void validateString(String parameter, String value)
+ throws ConfigurationException
+ {
+ String validValuesStr =
+ (String) deploymentProps.get(parameter + STRINGVALS);
+
+ if (validValuesStr != null) {
+ String[] validValues = validValuesStr.split(",");
+ if (!Arrays.asList(validValues).contains(value)) {
+ throw new ConfigurationException
+ ("invalid string parameter ["+parameter+"] in "
+ +"list ["+validValuesStr+"]");
+ }
+ }
+ return;
+ }
+
+ private static String[] validateStringArray(String parameter, String value)
+ throws ConfigurationException
+ {
+ String validValuesStr =
+ (String)(deploymentProps.get(parameter + STRINGVALS));
+ String[] values = value.split(",");
+
+ if (validValuesStr != null) {
+ String[] validValues = validValuesStr.split(",");
+ List validValuesList = Arrays.asList(validValues);
+ for (int i=0; i<values.length; i++) {
+ if (!validValuesList.contains(values[i])) {
+ throw new ConfigurationException
+ ("invalid string parameter ["+parameter+"] in "
+ +"list "+validValuesList);
+ }
+ }
+ }
+ return values;
+ }
+
+ private static int validateInt(String parameter, String strvalue)
+ throws ConfigurationException
+ {
+ String maxString = (String)(deploymentProps.get(parameter + MAX));
+ String minString = (String)(deploymentProps.get(parameter + MIN));
+
+ int value = str2int(strvalue);
+
+ if (maxString != null) {
+ int max = Integer.parseInt(maxString);
+ if (value > max) {
+ throw new ConfigurationException("parameter ["+parameter+"] "
+ +"exceeds maximum ["+max+"]");
+ }
+ }
+ return value;
+ }
+
+ private static long validateLong(String parameter, String strvalue)
+ throws ConfigurationException
+ {
+ String maxString = (String)(deploymentProps.get(parameter + MAX));
+ String minString = (String)(deploymentProps.get(parameter + MIN));
+
+ long value = str2long(strvalue);
+
+ if (maxString != null) {
+ long max = Long.parseLong(maxString);
+ if (value > max) {
+ throw new ConfigurationException("parameter ["+parameter+"] "
+ +"exceeds maximum ["+max+"]");
+ }
+ }
+ if (minString != null) {
+ long min = Long.parseLong(minString);
+ if (value < min) {
+ throw new ConfigurationException("parameter ["+parameter+"] "
+ +"is less than manimum "
+ +"["+min+"]");
+ }
+ }
+ return value;
+ }
+
+
+ private static String getPropertiesPath() {
+ String rootPath = "/opt/bigdata";//real installation
+ String appHome = System.getProperty("appHome");//pstart
+ String appDotHome = System.getProperty("app.home");//build.xml
+
+ if(appHome != null) {
+ rootPath = appHome;
+ } else if(appDotHome != null) {
+ rootPath = appDotHome + File.separator
+ + "dist" + File.separator + "bigdata";
+ }
+ String relPath = "var" + File.separator + "config"
+ + File.separator + "deploy";
+ String retPath = rootPath + File.separator + relPath;
+ //eclipse
+ if( !(new File(retPath)).exists() ) {
+ String tmpPath = "bigdata-jini" + File.separator + "src"
+ + File.separator + "java"
+ + File.separator + "com"
+ + File.separator + "bigdata"
+ + File.separator + "util"
+ + File.separator + "config";
+ retPath = (new File(tmpPath)).getAbsolutePath();
+ }
+ return retPath;
+ }
+
+ private static void loadDeployProps(Properties deployProps) {
+ loadDefaultProps(deployProps);
+ loadOverrideProps(deployProps);
+ }
+
+ private static void loadDefaultProps(Properties deployProps) {
+ FileInputStream fis = null;
+ try {
+ String flnm = getPropertiesPath()
+ + File.separator + "default-deploy.properties";
+ fis = new FileInputStream(flnm);
+ deployProps.load(fis);
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ } finally {
+ if (fis != null) {
+ try {
+ fis.close();
+ } catch (IOException ioex) { /* swallow */ }
+ }
+ }
+ }
+
+ private static void loadOverrideProps(Properties deployProps) {
+ FileInputStream fis = null;
+ try {
+ String flnm = getPropertiesPath()
+ + File.separator + "deploy.properties";
+ fis = new FileInputStream(new File(flnm));
+ deployProps.load(fis);
+ } catch (Exception ex) {
+ // using all defaults
+ } finally {
+ if (fis != null) {
+ try {
+ fis.close();
+ } catch (IOException ioex) { /* swallow */ }
+ }
+ }
+ }
+
+
+ private static int str2int(String argx) {
+ long l;
+
+ if( argx.trim().equals(Integer.MAX_VALUE) ) return Integer.MAX_VALUE;
+ if( argx.trim().equals(Integer.MIN_VALUE) ) return Integer.MIN_VALUE;
+
+ l = str2long(argx);
+ if (l < Integer.MAX_VALUE && l > Integer.MIN_VALUE) {
+ return (int) l;
+ } else {
+ throw new NumberFormatException("Invalid number:"+argx
+ +" --number out of range");
+ }
+ }
+
+ private static long str2long(String argx) {
+
+ int minDigitNumBetwnComma = 3;
+
+ String arg = argx.trim();
+ arg = arg.replaceAll("\"", ""); // strip all...
[truncated message content] |
|
From: <mrp...@us...> - 2010-07-23 21:02:20
|
Revision: 3282
http://bigdata.svn.sourceforge.net/bigdata/?rev=3282&view=rev
Author: mrpersonick
Date: 2010-07-23 21:02:07 +0000 (Fri, 23 Jul 2010)
Log Message:
-----------
added test cases for encoding / decoding extensions
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractIV.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractInlineIV.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ExtensionIV.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractIV.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractIV.java 2010-07-23 20:49:53 UTC (rev 3281)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractIV.java 2010-07-23 21:02:07 UTC (rev 3282)
@@ -597,6 +597,7 @@
* Note: We have to handle the unsigned byte, short, int and long values
* specially to get the correct total key order.
*/
+
final DTE dte = getDTE();
if (isBNode()) {
@@ -613,7 +614,9 @@
} else {
- final AbstractLiteralIV<?, ?> t = (AbstractLiteralIV<?, ?>) this;
+ final AbstractLiteralIV<?, ?> t = isExtension() ?
+ ((ExtensionIV) this).getDelegate() :
+ (AbstractLiteralIV<?, ?>) this;
switch (dte) {
case XSDBoolean:
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractInlineIV.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractInlineIV.java 2010-07-23 20:49:53 UTC (rev 3281)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractInlineIV.java 2010-07-23 21:02:07 UTC (rev 3282)
@@ -53,6 +53,13 @@
}
+ protected AbstractInlineIV(final VTE vte, final boolean extension,
+ final DTE dte) {
+
+ super(vte, true/* inline */, extension, dte);
+
+ }
+
/**
* Returns the String-value of a Value object. This returns either a
* Literal's label, a URI's URI or a BNode's ID.
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ExtensionIV.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ExtensionIV.java 2010-07-23 20:49:53 UTC (rev 3281)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ExtensionIV.java 2010-07-23 21:02:07 UTC (rev 3282)
@@ -2,7 +2,6 @@
import com.bigdata.rawstore.Bytes;
import com.bigdata.rdf.model.BigdataLiteral;
-import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.model.BigdataValueFactory;
public class ExtensionIV<V extends BigdataLiteral>
@@ -19,7 +18,7 @@
public ExtensionIV(final AbstractLiteralIV delegate,
final TermId datatype) {
- super(VTE.LITERAL, DTE.Extension);
+ super(VTE.LITERAL, true, delegate.getDTE());
this.delegate = delegate;
this.datatype = datatype;
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java 2010-07-23 20:49:53 UTC (rev 3281)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java 2010-07-23 21:02:07 UTC (rev 3282)
@@ -31,6 +31,8 @@
import java.math.BigInteger;
import java.util.Random;
import java.util.UUID;
+import org.openrdf.model.URI;
+import org.openrdf.model.impl.LiteralImpl;
import junit.framework.TestCase2;
@@ -41,6 +43,7 @@
import com.bigdata.rdf.model.BigdataURI;
import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.model.BigdataValueFactory;
+import com.bigdata.rdf.model.BigdataValueFactoryImpl;
/**
* Unit tests for encoding and decoding compound keys (such as are used by the
@@ -1024,4 +1027,50 @@
}
+ public void test_SPO_encodeDecodeEpoch() {
+
+ final BigdataValueFactory vf = BigdataValueFactoryImpl.getInstance("test");
+
+ EpochExtension<BigdataValue> ext = new EpochExtension<BigdataValue>();
+ ext.resolveDatatype(new IDatatypeURIResolver() {
+ public BigdataURI resolve(URI uri) {
+ BigdataURI buri = vf.createURI(uri.stringValue());
+ buri.setIV(new TermId(VTE.URI, 1024));
+ return buri;
+ }
+ });
+
+ final IV<?, ?>[] e = {//
+ new TermId<BigdataURI>(VTE.URI, 1L),//
+ new TermId<BigdataURI>(VTE.URI, 2L),//
+ ext.createIV(new LiteralImpl("1234", EpochExtension.EPOCH)),
+ };
+
+ doEncodeDecodeTest(e);
+
+ }
+
+ public void test_SPO_encodeDecodeColor() {
+
+ final BigdataValueFactory vf = BigdataValueFactoryImpl.getInstance("test");
+
+ ColorsEnumExtension<BigdataValue> ext = new ColorsEnumExtension<BigdataValue>();
+ ext.resolveDatatype(new IDatatypeURIResolver() {
+ public BigdataURI resolve(URI uri) {
+ BigdataURI buri = vf.createURI(uri.stringValue());
+ buri.setIV(new TermId(VTE.URI, 1024));
+ return buri;
+ }
+ });
+
+ final IV<?, ?>[] e = {//
+ new TermId<BigdataURI>(VTE.URI, 1L),//
+ new TermId<BigdataURI>(VTE.URI, 2L),//
+ ext.createIV(new LiteralImpl("Blue", ColorsEnumExtension.COLOR)),
+ };
+
+ doEncodeDecodeTest(e);
+
+ }
+
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2010-07-23 20:49:59
|
Revision: 3281
http://bigdata.svn.sourceforge.net/bigdata/?rev=3281&view=rev
Author: mrpersonick
Date: 2010-07-23 20:49:53 +0000 (Fri, 23 Jul 2010)
Log Message:
-----------
fixed a bnode bug
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractIV.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractIV.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractIV.java 2010-07-23 20:30:13 UTC (rev 3280)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractIV.java 2010-07-23 20:49:53 UTC (rev 3281)
@@ -598,54 +598,70 @@
* specially to get the correct total key order.
*/
final DTE dte = getDTE();
+
+ if (isBNode()) {
+
+ switch (dte) {
+ case XSDInt:
+ final int i = ((NumericBNodeIV) this).getInlineValue().intValue();
+ keyBuilder.append(i);
+ break;
+ case UUID:
+ keyBuilder.append((UUID) getInlineValue());
+ break;
+ }
+
+ } else {
- final AbstractLiteralIV<?, ?> t = (AbstractLiteralIV<?, ?>) this;
-
- switch (dte) {
- case XSDBoolean:
- keyBuilder.append((byte) (t.booleanValue() ? 1 : 0));
- break;
- case XSDByte:
- keyBuilder.append(t.byteValue());
- break;
- case XSDShort:
- keyBuilder.append(t.shortValue());
- break;
- case XSDInt:
- keyBuilder.append(t.intValue());
- break;
- case XSDLong:
- keyBuilder.append(t.longValue());
- break;
- case XSDFloat:
- keyBuilder.append(t.floatValue());
- break;
- case XSDDouble:
- keyBuilder.append(t.doubleValue());
- break;
- case XSDInteger:
- keyBuilder.append(t.integerValue());
- break;
- case XSDDecimal:
- keyBuilder.append(t.decimalValue());
- break;
- case UUID:
- keyBuilder.append((UUID)t.getInlineValue());
- break;
-// case XSDUnsignedByte:
-// keyBuilder.appendUnsigned(t.byteValue());
-// break;
-// case XSDUnsignedShort:
-// keyBuilder.appendUnsigned(t.shortValue());
-// break;
-// case XSDUnsignedInt:
-// keyBuilder.appendUnsigned(t.intValue());
-// break;
-// case XSDUnsignedLong:
-// keyBuilder.appendUnsigned(t.longValue());
-// break;
- default:
- throw new AssertionError(toString());
+ final AbstractLiteralIV<?, ?> t = (AbstractLiteralIV<?, ?>) this;
+
+ switch (dte) {
+ case XSDBoolean:
+ keyBuilder.append((byte) (t.booleanValue() ? 1 : 0));
+ break;
+ case XSDByte:
+ keyBuilder.append(t.byteValue());
+ break;
+ case XSDShort:
+ keyBuilder.append(t.shortValue());
+ break;
+ case XSDInt:
+ keyBuilder.append(t.intValue());
+ break;
+ case XSDLong:
+ keyBuilder.append(t.longValue());
+ break;
+ case XSDFloat:
+ keyBuilder.append(t.floatValue());
+ break;
+ case XSDDouble:
+ keyBuilder.append(t.doubleValue());
+ break;
+ case XSDInteger:
+ keyBuilder.append(t.integerValue());
+ break;
+ case XSDDecimal:
+ keyBuilder.append(t.decimalValue());
+ break;
+ case UUID:
+ keyBuilder.append((UUID)t.getInlineValue());
+ break;
+ // case XSDUnsignedByte:
+ // keyBuilder.appendUnsigned(t.byteValue());
+ // break;
+ // case XSDUnsignedShort:
+ // keyBuilder.appendUnsigned(t.shortValue());
+ // break;
+ // case XSDUnsignedInt:
+ // keyBuilder.appendUnsigned(t.intValue());
+ // break;
+ // case XSDUnsignedLong:
+ // keyBuilder.appendUnsigned(t.longValue());
+ // break;
+ default:
+ throw new AssertionError(toString());
+ }
+
}
return keyBuilder;
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java 2010-07-23 20:30:13 UTC (rev 3280)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java 2010-07-23 20:49:53 UTC (rev 3281)
@@ -35,6 +35,7 @@
import com.bigdata.rawstore.Bytes;
import com.bigdata.rdf.internal.constraints.AbstractInlineConstraint;
import com.bigdata.rdf.internal.constraints.InlineGT;
+import com.bigdata.rdf.model.BigdataBNode;
import com.bigdata.rdf.model.BigdataLiteral;
@@ -325,8 +326,12 @@
}
case XSDInt: {
final int x = KeyBuilder.decodeInt(key, o);
- final AbstractLiteralIV iv = new XSDIntIV<BigdataLiteral>(x);
- return isExtension ? new ExtensionIV(iv, datatype) : iv;
+ if (vte == VTE.LITERAL) {
+ final AbstractLiteralIV iv = new XSDIntIV<BigdataLiteral>(x);
+ return isExtension ? new ExtensionIV(iv, datatype) : iv;
+ } else {
+ return new NumericBNodeIV<BigdataBNode>(x);
+ }
}
case XSDLong: {
final long x = KeyBuilder.decodeLong(key, o);
@@ -355,8 +360,12 @@
}
case UUID: {
final UUID x = KeyBuilder.decodeUUID(key, o);
- final AbstractLiteralIV iv = new UUIDLiteralIV<BigdataLiteral>(x);
- return isExtension ? new ExtensionIV(iv, datatype) : iv;
+ if (vte == VTE.LITERAL) {
+ final AbstractLiteralIV iv = new UUIDLiteralIV<BigdataLiteral>(x);
+ return isExtension ? new ExtensionIV(iv, datatype) : iv;
+ } else {
+ return new UUIDBNodeIV<BigdataBNode>(x);
+ }
}
// case XSDUnsignedByte:
// keyBuilder.appendUnsigned(t.byteValue());
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java 2010-07-23 20:30:13 UTC (rev 3280)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java 2010-07-23 20:49:53 UTC (rev 3281)
@@ -36,6 +36,7 @@
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.rdf.model.BigdataBNode;
import com.bigdata.rdf.model.BigdataLiteral;
import com.bigdata.rdf.model.BigdataURI;
import com.bigdata.rdf.model.BigdataValue;
@@ -1009,5 +1010,18 @@
doEncodeDecodeTest(e);
}
+
+ public void test_SPO_encodeDecode_BNode() {
+
+ final IV<?, ?>[] e = {//
+ new TermId<BigdataURI>(VTE.URI, 1L),//
+ new TermId<BigdataURI>(VTE.URI, 2L),//
+ new UUIDBNodeIV<BigdataBNode>(UUID.randomUUID()),//
+ new NumericBNodeIV<BigdataBNode>(52),//
+ };
+ doEncodeDecodeTest(e);
+
+ }
+
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2010-07-23 20:30:19
|
Revision: 3280
http://bigdata.svn.sourceforge.net/bigdata/?rev=3280&view=rev
Author: mrpersonick
Date: 2010-07-23 20:30:13 +0000 (Fri, 23 Jul 2010)
Log Message:
-----------
fixing bnode bug
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-07-23 20:29:43 UTC (rev 3279)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-07-23 20:30:13 UTC (rev 3280)
@@ -391,8 +391,8 @@
AbstractTripleStore.Options.DEFAULT_INLINE_LITERALS));
inlineBNodes = storeBlankNodes && Boolean.parseBoolean(getProperty(
- AbstractTripleStore.Options.INLINE_LITERALS,
- AbstractTripleStore.Options.DEFAULT_INLINE_LITERALS));
+ AbstractTripleStore.Options.INLINE_BNODES,
+ AbstractTripleStore.Options.DEFAULT_INLINE_BNODES));
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2010-07-23 20:29:50
|
Revision: 3279
http://bigdata.svn.sourceforge.net/bigdata/?rev=3279&view=rev
Author: mrpersonick
Date: 2010-07-23 20:29:43 +0000 (Fri, 23 Jul 2010)
Log Message:
-----------
Added generics for extension classes
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java 2010-07-23 18:56:20 UTC (rev 3278)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java 2010-07-23 20:29:43 UTC (rev 3279)
@@ -29,6 +29,7 @@
import org.openrdf.model.Value;
import org.openrdf.model.impl.URIImpl;
import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.model.BigdataValueFactory;
import com.bigdata.rdf.store.BD;
@@ -36,7 +37,7 @@
* Example of how to do a custom enum and map that enum over a byte using a
* native inline {@link XSDByteIV}.
*/
-public class ColorsEnumExtension implements IExtension {
+public class ColorsEnumExtension<V extends BigdataValue> implements IExtension<V> {
/**
* The datatype URI for the colors enum extension.
@@ -99,7 +100,7 @@
* a {@link Color}, and then use the string value of the {@link Color} to
* create an RDF literal.
*/
- public Value asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
+ public V asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
final byte b = iv.getDelegate().byteValue();
@@ -108,7 +109,7 @@
if (c == null)
throw new RuntimeException("bad color got encoded somehow");
- return vf.createLiteral(c.toString(), color);
+ return (V) vf.createLiteral(c.toString(), color);
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java 2010-07-23 18:56:20 UTC (rev 3278)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java 2010-07-23 20:29:43 UTC (rev 3279)
@@ -30,6 +30,7 @@
import org.openrdf.model.datatypes.XMLDatatypeUtil;
import org.openrdf.model.impl.URIImpl;
import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.model.BigdataValueFactory;
import com.bigdata.rdf.store.BD;
@@ -38,7 +39,7 @@
* that represent time in milliseconds since the epoch. The milliseconds are
* encoded as an inline long.
*/
-public class EpochExtension implements IExtension {
+public class EpochExtension<V extends BigdataValue> implements IExtension<V> {
/**
* The datatype URI for the epoch extension.
@@ -100,9 +101,9 @@
* string value of the native type) to create a literal with the epoch
* datatype.
*/
- public Value asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
+ public V asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
- return vf.createLiteral(iv.stringValue(), epoch);
+ return (V) vf.createLiteral(iv.stringValue(), epoch);
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java 2010-07-23 18:56:20 UTC (rev 3278)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java 2010-07-23 20:29:43 UTC (rev 3279)
@@ -26,6 +26,7 @@
import org.openrdf.model.Value;
import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.model.BigdataValueFactory;
/**
@@ -37,7 +38,7 @@
* the datatype URI it needs resolved and the resolver will lookup (or create)
* the {@link TermId}.
*/
-public interface IExtension {
+public interface IExtension<V extends BigdataValue> {
/**
* This will be called very early in the IExtension lifecycle so that the
@@ -77,6 +78,6 @@
* @return
* the RDF value
*/
- Value asValue(final ExtensionIV iv, final BigdataValueFactory vf);
+ V asValue(final ExtensionIV iv, final BigdataValueFactory vf);
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java 2010-07-23 18:56:20 UTC (rev 3278)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java 2010-07-23 20:29:43 UTC (rev 3279)
@@ -33,7 +33,7 @@
* Configuration determines which RDF Values are inlined into the statement
* indices rather than being assigned term identifiers by the lexicon.
*/
-public interface ILexiconConfiguration {
+public interface ILexiconConfiguration<V extends BigdataValue> {
/**
* Create an inline {@link IV} for the supplied RDF value if inlining is
@@ -61,7 +61,7 @@
* @return
* the RDF value
*/
- Value asValue(final ExtensionIV iv, final BigdataValueFactory vf);
+ V asValue(final ExtensionIV iv, final BigdataValueFactory vf);
/**
* <code>true</code> iff the <code>vte</code> and <code>dte</code>
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java 2010-07-23 18:56:20 UTC (rev 3278)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java 2010-07-23 20:29:43 UTC (rev 3279)
@@ -50,7 +50,8 @@
* @todo large literal size boundary.
* @todo other configuration options.
*/
-public class LexiconConfiguration implements ILexiconConfiguration {
+public class LexiconConfiguration<V extends BigdataValue>
+ implements ILexiconConfiguration {
private boolean inlineLiterals, inlineBNodes;
@@ -76,9 +77,9 @@
}
- public Value asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
+ public V asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
final TermId datatype = iv.getExtensionDatatype();
- return termIds.get(datatype).asValue(iv, vf);
+ return (V) termIds.get(datatype).asValue(iv, vf);
}
public IV createInlineIV(final Value value) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2010-07-23 18:56:26
|
Revision: 3278
http://bigdata.svn.sourceforge.net/bigdata/?rev=3278&view=rev
Author: mrpersonick
Date: 2010-07-23 18:56:20 +0000 (Fri, 23 Jul 2010)
Log Message:
-----------
cleaning up, documenting
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IDatatypeURIResolver.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IDatatypeURIResolver.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IDatatypeURIResolver.java 2010-07-23 18:53:48 UTC (rev 3277)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IDatatypeURIResolver.java 2010-07-23 18:56:20 UTC (rev 3278)
@@ -34,15 +34,15 @@
* Returns a fully resolved datatype URI with the {@link TermId} set.
* {@link IExtension}s will handle encoding and decoding of inline literals
* for custom datatypes, however to do so they will need the term identifier
- * for the custom datatype. By passing an instance of this interface into
- * the constructor for the {@link IExtension}, it will be able to resolve
- * its datatype URI and cache it for future use.
+ * for the custom datatype. By passing an instance of this interface to
+ * the {@link IExtension}, it will be able to resolve its datatype URI and
+ * cache it for future use.
* <p>
* If the datatype URI is not already in the lexicon this method MUST add
* it to the lexicon so that it has an assigned term identifier.
* <p>
* This is implemented by the {@link LexiconRelation}.
- *
+ *
* @param uri
* the term to resolve
* @return
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2010-07-23 18:53:54
|
Revision: 3277
http://bigdata.svn.sourceforge.net/bigdata/?rev=3277&view=rev
Author: mrpersonick
Date: 2010-07-23 18:53:48 +0000 (Fri, 23 Jul 2010)
Log Message:
-----------
cleaning up, documenting
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java 2010-07-23 18:43:00 UTC (rev 3276)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java 2010-07-23 18:53:48 UTC (rev 3277)
@@ -32,9 +32,15 @@
import com.bigdata.rdf.model.BigdataValueFactory;
import com.bigdata.rdf.store.BD;
-
+/**
+ * Example of how to do a custom enum and map that enum over a byte using a
+ * native inline {@link XSDByteIV}.
+ */
public class ColorsEnumExtension implements IExtension {
+ /**
+ * The datatype URI for the colors enum extension.
+ */
public static final URI COLOR = new URIImpl(BD.NAMESPACE + "Color");
private BigdataURI color;
@@ -55,6 +61,15 @@
}
+ /**
+ * Attempts to convert the supplied RDF value into a colors enum
+ * representation. Tests for a literal value with the correct datatype
+ * that can be converted to one of the colors in the {@link Color} enum
+ * based on the string value of the literal's label. Each {@link Color}
+ * in the enum maps to a particular byte. This byte is encoded in a
+ * delegate {@link XSDByteIV}, and an {@link ExtensionIV} is returned that
+ * wraps the native type.
+ */
public ExtensionIV createIV(final Value value) {
if (value instanceof Literal == false)
@@ -79,12 +94,20 @@
}
+ /**
+ * Attempt to convert the {@link AbstractLiteralIV#byteValue()} back into
+ * a {@link Color}, and then use the string value of the {@link Color} to
+ * create an RDF literal.
+ */
public Value asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
final byte b = iv.getDelegate().byteValue();
final Color c = Color.valueOf(b);
+ if (c == null)
+ throw new RuntimeException("bad color got encoded somehow");
+
return vf.createLiteral(c.toString(), color);
}
@@ -110,13 +133,6 @@
}
static final public Color valueOf(final byte b) {
- /*
- * Note: This switch MUST correspond to the declarations above (you can
- * not made the cases of the switch from [v] since it is not considered
- * a to be constant by the compiler).
- *
- * Note: This masks off everything but the lower 4 bits.
- */
switch (b) {
case 0:
return Red;
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java 2010-07-23 18:43:00 UTC (rev 3276)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java 2010-07-23 18:53:48 UTC (rev 3277)
@@ -33,15 +33,21 @@
import com.bigdata.rdf.model.BigdataValueFactory;
import com.bigdata.rdf.store.BD;
-
+/**
+ * This implementation of {@link IExtension} implements inlining for literals
+ * that represent time in milliseconds since the epoch. The milliseconds are
+ * encoded as an inline long.
+ */
public class EpochExtension implements IExtension {
+ /**
+ * The datatype URI for the epoch extension.
+ */
public static final URI EPOCH = new URIImpl(BD.NAMESPACE + "Epoch");
private BigdataURI epoch;
public EpochExtension() {
-
}
public void resolveDatatype(final IDatatypeURIResolver resolver) {
@@ -56,6 +62,13 @@
}
+ /**
+ * Attempts to convert the supplied value into an epoch representation.
+ * Tests for a literal value with the correct datatype that can be converted
+ * to a positive long integer. Encodes the long in a delegate
+ * {@link XSDLongIV}, and returns an {@link ExtensionIV} to wrap the native
+ * type.
+ */
public ExtensionIV createIV(final Value value) {
if (value instanceof Literal == false)
@@ -63,8 +76,9 @@
final Literal lit = (Literal) value;
- if (lit.getDatatype() == null ||
- !EPOCH.stringValue().equals(lit.getDatatype().stringValue()))
+ final URI dt = lit.getDatatype();
+
+ if (dt == null || !EPOCH.stringValue().equals(dt.stringValue()))
throw new IllegalArgumentException();
final String s = value.stringValue();
@@ -81,6 +95,11 @@
}
+ /**
+ * Use the string value of the {@link ExtensionIV} (which defers to the
+ * string value of the native type) to create a literal with the epoch
+ * datatype.
+ */
public Value asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
return vf.createLiteral(iv.stringValue(), epoch);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <btm...@us...> - 2010-07-23 18:43:07
|
Revision: 3276
http://bigdata.svn.sourceforge.net/bigdata/?rev=3276&view=rev
Author: btmurphy
Date: 2010-07-23 18:43:00 +0000 (Fri, 23 Jul 2010)
Log Message:
-----------
merge -r3143::HEAD(3267) ~/bigdata/trunk ~/bigdata/branches/dev-btm [trunk --> branch dev-btm]
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestICUUnicodeKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestJDKUnicodeKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestRingBuffer.java
branches/dev-btm/bigdata/src/test/com/bigdata/sparse/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/sparse/TestKeyEncodeDecode.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/MonitorCreatePhysicalServiceLocksTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/TruthMaintenance.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNamedGraphs.java
branches/dev-btm/build.xml
branches/dev-btm/src/resources/config/bigdataCluster.config
branches/dev-btm/src/resources/config/bigdataCluster16.config
branches/dev-btm/src/resources/config/standalone/bigdataStandalone.config
Added Paths:
-----------
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java
Removed Paths:
-------------
branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractUnicodeKeyBuilderTestCase.java
Property Changed:
----------------
branches/dev-btm/
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/
branches/dev-btm/dsi-utils/
branches/dev-btm/dsi-utils/src/test/
branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/src/resources/bin/config/
Property changes on: branches/dev-btm
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/bugfix-btm:2594-2779
/trunk:2575-2594,2596-2877,2882-2903,2910-3143
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3269
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -10,7 +10,8 @@
/**
* The JDK bundles support for generating Unicode sort keys, but that
- * support does NOT include compressed sort keys.
+ * support does NOT include compressed sort keys and embeds <code>nul</code>
+ * bytes into its Unicode sort keys.
*/
JDK,
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -144,6 +144,7 @@
/**
* Representation includes all aspects of the {@link Serializable} state.
*/
+ @Override
public String toString() {
StringBuilder sb = new StringBuilder(getClass().getName());
@@ -270,14 +271,10 @@
/*
* Figure out which collator to use.
- *
- * Note: The default depends on whether or not the ICU library is on
- * the class path. When it is, we always default to the ICU library.
*/
collator = CollatorEnum.valueOf(getProperty(properties,
- Options.COLLATOR, (icu_avail ? CollatorEnum.ICU.toString()
- : CollatorEnum.JDK.toString())));
+ Options.COLLATOR, CollatorEnum.ICU.toString()));
// true iff the collator was _explicitly_ specified.
final boolean explicitCollatorChoice = getProperty(properties,
@@ -349,7 +346,7 @@
* Figure out the collator strength.
*/
- Object strength = null;
+ Object tmpStrength = null;
final String val = getProperty(properties, Options.STRENGTH);
@@ -357,24 +354,24 @@
try {
- strength = StrengthEnum.valueOf(val);
+ tmpStrength = StrengthEnum.valueOf(val);
} catch (RuntimeException ex) {
- strength = Integer.parseInt(val);
+ tmpStrength = Integer.parseInt(val);
}
}
if (log.isInfoEnabled())
- log.info(Options.STRENGTH + "=" + strength);
+ log.info(Options.STRENGTH + "=" + tmpStrength);
/*
* Note: MAY be null (when null, does not override the collator's
* default).
*/
- this.strength = strength;
+ this.strength = tmpStrength;
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -28,7 +28,6 @@
import java.util.Locale;
-import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import com.ibm.icu.text.Collator;
@@ -90,18 +89,6 @@
protected static final Logger log = Logger.getLogger(ICUSortKeyGenerator.class);
/**
- * True iff the {@link #log} level is INFO or less.
- */
- final protected boolean INFO = log.getEffectiveLevel().toInt() <= Level.INFO
- .toInt();
-
- /**
- * True iff the {@link #log} level is DEBUG or less.
- */
- final protected boolean DEBUG = log.getEffectiveLevel().toInt() <= Level.DEBUG
- .toInt();
-
- /**
* Used to encode unicode strings into compact byte[]s that have the same
* sort order (aka sort keys).
*/
@@ -128,7 +115,7 @@
this.locale = locale;
- if(INFO) log.info("locale="+locale);
+ if(log.isInfoEnabled()) log.info("locale="+locale);
this.collator = (RuleBasedCollator) Collator.getInstance(locale);
@@ -138,7 +125,7 @@
final int str = ((Integer) strength).intValue();
- if (INFO)
+ if (log.isInfoEnabled())
log.info("strength=" + str);
collator.setStrength(str);
@@ -147,7 +134,7 @@
StrengthEnum str = (StrengthEnum) strength;
- if (INFO)
+ if (log.isInfoEnabled())
log.info("strength=" + str);
switch (str) {
@@ -184,7 +171,7 @@
if (mode != null) {
- if(INFO) log.info("mode="+mode);
+ if(log.isInfoEnabled()) log.info("mode="+mode);
switch (mode) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -56,8 +56,8 @@
}
- public JDKSortKeyGenerator(Locale locale, Object strength,
- DecompositionEnum mode) {
+ public JDKSortKeyGenerator(final Locale locale, final Object strength,
+ final DecompositionEnum mode) {
if (locale == null)
throw new IllegalArgumentException();
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -180,9 +180,9 @@
* The buffer reference is used directly rather than making a
* copy of the data.
*/
- protected KeyBuilder(UnicodeSortKeyGenerator sortKeyGenerator, int len,
- byte[] buf) {
-
+ protected KeyBuilder(final UnicodeSortKeyGenerator sortKeyGenerator,
+ final int len, final byte[] buf) {
+
if (len < 0)
throw new IllegalArgumentException("len");
@@ -350,7 +350,7 @@
* The object responsible for generating sort keys from Unicode strings.
*
* The {@link UnicodeSortKeyGenerator} -or- <code>null</code> if Unicode
- * is not supported by this {@link KeyBuilder} instance.
+ * is not supported by this {@link IKeyBuilder} instance.
*/
final public UnicodeSortKeyGenerator getSortKeyGenerator() {
@@ -391,11 +391,11 @@
public KeyBuilder appendASCII(final String s) {
- int len = s.length();
+ int tmpLen = s.length();
- ensureFree(len);
+ ensureFree(tmpLen);
- for(int j=0; j<len; j++) {
+ for(int j=0; j<tmpLen; j++) {
char ch = s.charAt(j);
@@ -1356,6 +1356,11 @@
}
+ /**
+ * Create an instance for ASCII keys.
+ *
+ * @return The new instance.
+ */
public static IKeyBuilder newInstance() {
return newInstance(DEFAULT_INITIAL_CAPACITY);
@@ -1393,10 +1398,8 @@
/**
* Optional property specifies the library that will be used to generate
- * sort keys from Unicode data. The default always supports Unicode, but
- * the library choice depends on whether or not ICU library is found on
- * the classpath. When the ICU library is present, it is the default.
- * Otherwise the JDK library is the default. You may explicitly specify
+ * sort keys from Unicode data. The ICU library is the default.
+ * You may explicitly specify
* the library choice using one of the {@link CollatorEnum} values. The
* {@link CollatorEnum#ASCII} value may be used to disable Unicode
* support entirely, treating the characters as if they were ASCII. If
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -149,6 +149,7 @@
// allow client requests to finish normally.
new ShutdownHelper(threadPool, 10L/*logTimeout*/, TimeUnit.SECONDS) {
+ @Override
public void logTimeout() {
log.warn("Awaiting thread pool termination: elapsed="
@@ -170,6 +171,7 @@
new ShutdownHelper(scheduledExecutorService, 10L/* logTimeout */,
TimeUnit.SECONDS) {
+ @Override
public void logTimeout() {
log.warn("Awaiting sample service termination: elapsed="
@@ -650,12 +652,7 @@
TimeUnit.MILLISECONDS // unit
);
- addScheduledTask(//
- new StartDeferredTasksTask(),// task to run.
- 150, // initialDelay (ms)
- 2000, // delay
- TimeUnit.MILLISECONDS // unit
- );
+ getExecutorService().execute(new StartDeferredTasksTask());
// Setup locator.
resourceLocator = new DefaultResourceLocator(this,
@@ -888,12 +885,12 @@
*/
public boolean isServiceReady() {
- final AbstractClient<T> client = this.client;
+ final AbstractClient<T> thisClient = this.client;
- if (client == null)
+ if (thisClient == null)
return false;
- final IFederationDelegate<T> delegate = client.getDelegate();
+ final IFederationDelegate<T> delegate = thisClient.getDelegate();
if (delegate == null)
return false;
@@ -963,11 +960,11 @@
}
// @todo really, we should test like this everywhere.
- final AbstractClient client = this.client;
+ final AbstractClient thisClient = this.client;
- if (client != null && client.isConnected()) {
+ if (thisClient != null && thisClient.isConnected()) {
- client.getDelegate().serviceLeave(serviceUUID);
+ thisClient.getDelegate().serviceLeave(serviceUUID);
}
@@ -996,7 +993,7 @@
static private String ERR_SERVICE_NOT_READY = "Service is not ready yet.";
/**
- * This task runs periodically. Once {@link #getServiceUUID()} reports a
+ * This task runs once. Once {@link #getServiceUUID()} reports a
* non-<code>null</code> value, it will start an (optional)
* {@link AbstractStatisticsCollector}, an (optional) httpd service, and
* the (required) {@link ReportTask}.
@@ -1008,8 +1005,6 @@
* {@link LoadBalancer} service know which services exist, which is
* important for some of its functions.
* <p>
- * Once these task(s) have been started, this task will throw an exception
- * in order to prevent it from being re-executed.
*
* FIXME This should explicitly await jini registrar discovery, zookeeper
* client connected, and whatever other preconditions must be statisified
@@ -1032,8 +1027,7 @@
*/
final long begin = System.currentTimeMillis();
- public StartDeferredTasksTask() {
-
+ private StartDeferredTasksTask() {
}
/**
@@ -1043,8 +1037,6 @@
*/
public void run() {
- final boolean started;
-
try {
// /*
@@ -1061,9 +1053,7 @@
// return;
//
// }
-
- started = startDeferredTasks();
-
+ startDeferredTasks();
} catch (Throwable t) {
log.warn("Problem in report task?", t);
@@ -1072,60 +1062,56 @@
}
- if (started) {
-
- /*
- * Note: This exception is thrown once this task has executed
- * successfully.
- */
-
- throw new RuntimeException("Normal completion.");
-
- }
-
}
/**
* Starts performance counter collection once the service {@link UUID}
* is known.
*
- * @return <code>true</code> iff performance counter collection was
- * started.
- *
* @throws IOException
* if {@link IDataService#getServiceUUID()} throws this
* exception (it never should since it is a local method
* call).
*/
- protected boolean startDeferredTasks() throws IOException {
+ protected void startDeferredTasks() throws IOException {
// elapsed time since we started running this task.
final long elapsed = System.currentTimeMillis() - begin;
- if (getServiceUUID() == null) {
-
+ // Wait for the service ID to become available, trying every
+ // two seconds, while logging failures.
+ while (true) {
+ if (getServiceUUID() != null) {
+ break;
+ }
if (elapsed > 1000 * 10)
log.warn(ERR_NO_SERVICE_UUID + " : iface="
+ getServiceIface() + ", name=" + getServiceName()
+ ", elapsed=" + elapsed);
else if (log.isInfoEnabled())
- log.info(ERR_NO_SERVICE_UUID);
-
- return false;
-
+ log.info(ERR_NO_SERVICE_UUID);
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ }
}
- if (!isServiceReady()) {
-
+ // Wait for the service to become ready, trying every
+ // two seconds, while logging failures.
+ while (true) {
+ if (isServiceReady()) {
+ break;
+ }
if (elapsed > 1000 * 10)
log.warn(ERR_SERVICE_NOT_READY + " : iface="
+ getServiceIface() + ", name=" + getServiceName()
+ ", elapsed=" + elapsed);
else if (log.isInfoEnabled())
log.info(ERR_SERVICE_NOT_READY + " : " + elapsed);
-
- return false;
-
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ }
}
/*
@@ -1153,9 +1139,6 @@
// notify delegates that deferred startup has occurred.
AbstractFederation.this.didStart();
-
- return true;
-
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -32,8 +32,12 @@
import org.apache.log4j.Logger;
+import com.bigdata.btree.DefaultTupleSerializer;
import com.bigdata.btree.IIndex;
import com.bigdata.btree.IndexMetadata;
+import com.bigdata.btree.keys.ASCIIKeyBuilderFactory;
+import com.bigdata.btree.keys.CollatorEnum;
+import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.journal.IIndexManager;
import com.bigdata.journal.ITx;
import com.bigdata.journal.TimestampUtility;
@@ -99,6 +103,25 @@
indexMetadata
.setSplitHandler(LogicalRowSplitHandler.INSTANCE);
+ if (CollatorEnum.JDK.toString().equals(
+ System.getProperty(KeyBuilder.Options.COLLATOR))) {
+ /*
+ * The JDK RulesBasedCollator embeds nul bytes in the
+ * Unicode sort keys. This makes them unsuitable for the
+ * SparseRowStore, which can not locate the start of the
+ * column name if there are embedded nuls in a Unicode
+ * primary key. As a work around, this forces an ASCII
+ * collation sequence if the JDK collator is the
+ * default. This is not ideal since non-ascii
+ * distinctions will be lost, but it is better than
+ * being unable to decode the column names.
+ */
+ log.warn("Forcing ASCII collator.");
+ indexMetadata
+ .setTupleSerializer(new DefaultTupleSerializer(
+ new ASCIIKeyBuilderFactory()));
+ }
+
// Register the index.
indexManager.registerIndex(indexMetadata);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -49,17 +49,22 @@
* prefix.
* <p>
* The encoded schema name is followed by the {@link KeyType#getByteCode()} and
- * then by a <code>nul</code> byte. By searching for the <code>nul</code>
- * byte we can identify the end of the encoded schema name and also the data
- * type of the primary key. Most kinds of primary keys have a fixed length
- * encoding, e.g., {@link Long}, {@link Double}, etc. However, Unicode primary
- * keys have a variable length encoding which makes life more ... complex. Since
- * the keys need to reflect the total sort order we can not include the byte
- * count of the primary key in the key itself. The only reasonable approach is
- * to append a byte sequence to the key that never occurs within the generated
- * Unicode sort keys. We use a <code>nul</code> byte for this purpose since it
- * is not emitted by most Unicode collation implementations as it would cause
- * grief for C-language strings.
+ * then by a <code>nul</code> byte. By searching for the <code>nul</code> byte
+ * we can identify the end of the encoded schema name and also the data type of
+ * the primary key. Most kinds of primary keys have a fixed length encoding,
+ * e.g., {@link Long}, {@link Double}, etc.
+ * <p>
+ * Unicode primary keys have a variable length encoding which makes life more
+ * complex. For Unicode primary keys, we break with the collation order and use
+ * the UTF8 encoding of the key. This means that the primary key can be decoded
+ * and preserves hierarchical namespace clustering within the row store but does
+ * not impose a total sort order per Unicode sort key semantics. The only
+ * reasonable approach is to append a byte sequence to the key that never occurs
+ * within the generated Unicode sort keys. Again, we use a <code>nul</code> byte
+ * to mark the end of the Unicode primary key since it is not emitted by most
+ * Unicode collation implementations as it would cause grief for C-language
+ * strings. (However, see SparseRowStore.Options#PRIMARY_KEY_UNICODE_CLEAN} for
+ * information on backward compatibility.)
*
* @see Schema#fromKey(IKeyBuilder, Object)
* @see KeyType#getKeyType(byte)
@@ -166,15 +171,15 @@
* The decoded primary key.
*
* @throws UnsupportedOperationException
- * if the primary key can not be decoded (e.g., for
- * {@link KeyType#Unicode} keys).
+ * if the primary key can not be decoded.
*/
public Object getPrimaryKey() {
if(primaryKey == null) {
-
- throw new UnsupportedOperationException("Can not decode: keyType="+primaryKeyType);
-
+
+ throw new UnsupportedOperationException("Can not decode: keyType="
+ + primaryKeyType);
+
}
return primaryKey;
@@ -220,14 +225,13 @@
* Note: the KeyType byte occurs after the schema name bytes and before
* the [nul].
*/
+ int primaryKeyOffset = 0;
{
boolean found = false;
int schemaBytesLength = 0;
- int primaryKeyOffset = 0;
-
for (int i = 0; i < key.length; i++) {
if (key[i] == (byte) 0) {
@@ -256,7 +260,6 @@
this.primaryKeyTypeOffset = schemaBytesLength;
- this.primaryKeyOffset = primaryKeyOffset;
// note: ArrayIndexOutOfBounds with index==-1 means ICU library not on classpath!
this.primaryKeyType = KeyType.getKeyType(KeyBuilder.decodeByte(key[primaryKeyTypeOffset]));
@@ -273,42 +276,48 @@
primaryKeyLength = primaryKeyType.getEncodedLength();
+ this.primaryKeyOffset = primaryKeyOffset;
+
columnNameOffset = primaryKeyOffset + primaryKeyLength;
} else {
/*
- * Scan for the next [nul] byte.
+ * Scan for the next [nul] byte (ASCII).
*/
boolean found = false;
int primaryKeyLength = 0;
-
+
for (int i = primaryKeyOffset; i < key.length; i++) {
if (key[i] == (byte) 0) {
primaryKeyLength = i - primaryKeyOffset;
-
+
found = true;
-
+
break;
-
+
}
}
-
- if(!found) {
+ if (!found) {
+
throw new RuntimeException(
"Could not locate the end of the encoded schema name: keyType="
- + primaryKeyType+", key="+BytesUtil.toString(key));
+ + primaryKeyType + ", key="
+ + BytesUtil.toString(key));
}
this.primaryKeyLength = primaryKeyLength;
- // Note: also skips the [nul] byte terminating the primary key.
+ this.primaryKeyOffset = primaryKeyOffset;
+
+ // Note: also skips the [nul] byte terminating the primary
+ // key.
this.columnNameOffset = primaryKeyOffset + primaryKeyLength + 1;
}
@@ -327,10 +336,26 @@
primaryKey = KeyBuilder.decodeFloat(key, primaryKeyOffset);
break;
case Unicode:
- /*
- * Note: Decode is not possible for this case.
- */
- primaryKey = null;
+ if (SparseRowStore.primaryKeyUnicodeClean) {
+ final byte[] bytes = new byte[primaryKeyLength];
+ System.arraycopy(key, primaryKeyOffset, bytes, 0, primaryKeyLength);
+ try {
+ primaryKey = new String(bytes, SparseRowStore.UTF8);
+ } catch (UnsupportedEncodingException ex) {
+ throw new RuntimeException(
+ "Could not decode the primary key"
+ + ": primaryKeyOffset="
+ + primaryKeyOffset
+ + ", primaryKeyLength="
+ + primaryKeyLength + ", key="
+ + BytesUtil.toString(key));
+ }
+ } else {
+ /*
+ * Note: Decode is not possible for this case.
+ */
+ primaryKey = null;
+ }
break;
case ASCII:
primaryKey = KeyBuilder.decodeASCII(key, primaryKeyOffset,
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -154,7 +154,7 @@
/*
* Key builder stuff.
*/
-
+
/**
* Helper method appends a typed value to the compound key (this is used to
* get the primary key into the compound key).
@@ -172,8 +172,9 @@
*
* @see KeyDecoder
*/
- final protected IKeyBuilder appendPrimaryKey(IKeyBuilder keyBuilder, Object v, boolean successor) {
-
+ final protected IKeyBuilder appendPrimaryKey(final IKeyBuilder keyBuilder,
+ final Object v, final boolean successor) {
+
final KeyType keyType = getPrimaryKeyType();
if (successor) {
@@ -187,9 +188,27 @@
case Float:
return keyBuilder.append(successor(keyBuilder,((Number) v).floatValue()));
case Double:
- return keyBuilder.append(successor(keyBuilder,((Number) v).doubleValue()));
- case Unicode:
- return keyBuilder.appendText(v.toString(), true/*unicode*/, true/*successor*/).appendNul();
+ return keyBuilder.append(successor(keyBuilder, ((Number) v)
+ .doubleValue()));
+ case Unicode: {
+ final String tmp = v.toString();
+ if (SparseRowStore.primaryKeyUnicodeClean) {
+ try {
+ keyBuilder.append(
+ SuccessorUtil.successor(tmp
+ .getBytes(SparseRowStore.UTF8)))
+ .appendNul();
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ // primary key in backwards compatibility mode.
+ keyBuilder
+ .appendText(tmp, true/* unicode */, true/* successor */)
+ .appendNul();
+ }
+ return keyBuilder;
+ }
case ASCII:
return keyBuilder.appendText(v.toString(), false/*unicode*/, true/*successor*/).appendNul();
case Date:
@@ -212,8 +231,22 @@
return keyBuilder.append(((Number) v).floatValue());
case Double:
return keyBuilder.append(((Number) v).doubleValue());
- case Unicode:
- return keyBuilder.appendText(v.toString(),true/*unicode*/,false/*successor*/).appendNul();
+ case Unicode: {
+ final String tmp = v.toString();
+ if (SparseRowStore.primaryKeyUnicodeClean) {
+ try {
+ keyBuilder.append(tmp.getBytes(SparseRowStore.UTF8))
+ .appendNul();
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ // primary key in backwards compatibility mode.
+ keyBuilder.appendText(v.toString(), true/* unicode */,
+ false/* successor */).appendNul();
+ }
+ return keyBuilder;
+ }
case ASCII:
return keyBuilder.appendText(v.toString(),false/*unicode*/,false/*successor*/).appendNul();
case Date:
@@ -243,8 +276,8 @@
* {@link #toKey(Object)}, which correctly forms the successor
* key in all cases.
*/
- final private Object successor(IKeyBuilder keyBuilder,Object v) {
-
+ final private Object successor(final IKeyBuilder keyBuilder, final Object v) {
+
final KeyType keyType = getPrimaryKeyType();
switch(keyType) {
@@ -290,7 +323,8 @@
*
* @see KeyDecoder
*/
- final protected IKeyBuilder fromKey(IKeyBuilder keyBuilder,Object primaryKey) {
+ final protected IKeyBuilder fromKey(final IKeyBuilder keyBuilder,
+ final Object primaryKey) {
keyBuilder.reset();
@@ -319,8 +353,9 @@
*
* @return
*/
- final public byte[] getPrefix(IKeyBuilder keyBuilder,Object primaryKey) {
-
+ final public byte[] getPrefix(final IKeyBuilder keyBuilder,
+ final Object primaryKey) {
+
return fromKey(keyBuilder, primaryKey).getKey();
}
@@ -384,8 +419,8 @@
* @throws IllegalArgumentException
* if <i>col</i> is not valid as the name of a column.
*/
- public byte[] getKey(IKeyBuilder keyBuilder, Object primaryKey, String col,
- long timestamp) {
+ public byte[] getKey(final IKeyBuilder keyBuilder, final Object primaryKey,
+ final String col, final long timestamp) {
if (keyBuilder == null)
throw new IllegalArgumentException();
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -24,6 +24,7 @@
*/
package com.bigdata.sparse;
+import java.text.RuleBasedCollator;
import java.util.Iterator;
import java.util.Map;
@@ -33,7 +34,9 @@
import com.bigdata.btree.IIndex;
import com.bigdata.btree.IRangeQuery;
import com.bigdata.btree.ITuple;
+import com.bigdata.btree.IndexMetadata;
import com.bigdata.btree.filter.FilterConstructor;
+import com.bigdata.btree.keys.CollatorEnum;
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.journal.ITimestampService;
@@ -186,6 +189,17 @@
* md.setSplitHandler(LogicalRowSplitHandler.INSTANCE);
* </pre>
*
+ * Note: The JDK {@link RuleBasedCollator} embeds <code>nul</code> bytes in
+ * the Unicode sort keys. This makes them unsuitable for the row store which
+ * can not locate the start of the column name if there are embedded
+ * <code>nul</code>s in the primaryKey. Therefore, if you are using the
+ * {@link CollatorEnum#JDK} as your default collator, then you MUST override
+ * the {@link IndexMetadata} for the row store to use either an ASCII
+ * collator or the ICU collator. In general, the ICU collator is superior to
+ * the JDK collator and will be used by default. The ASCII collator is not
+ * ideal since non-ascii distinctions will be lost, but it is better than
+ * being unable to decode the data in the row store.
+ *
* @param ndx
* The index.
*/
@@ -1029,4 +1043,53 @@
}
+ /**
+ * Options for the {@link SparseRowStore}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan
+ * Thompson</a>
+ * @version $Id$
+ */
+ public interface Options {
+
+ /**
+ * The primary key was originally written using a Unicode sort key.
+ * However, the JDK generates Unicode sort keys with embedded nuls and
+ * that broke the logic to detect the end of the Unicode primary keys.
+ * In order to accommodate this behavior, the Unicode primary key is now
+ * encoded as UTF8 which also has the advantage that we can decode
+ * Unicode primary keys. Standard prefix compression on the B+Tree
+ * should make up for the larger representation of the Unicode primary
+ * key in the B+Tree.
+ * <p>
+ * This change was introduced on 7/15/2010 in the trunk and breaks
+ * compatibility with earlier revisions of the {@link SparseRowStore}.
+ * This flag may be set to <code>false</code> for backward
+ * compatibility.
+ *
+ * @see Options#DEFAULT_PRIMARY_KEY_UNICODE_CLEAN
+ */
+ String PRIMARY_KEY_UNICODE_CLEAN = Schema.class.getName()
+ + ".primaryKey.unicodeClean";
+
+ /**
+ * FIXME Change over the [true] for the next release.
+ *
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/107
+ */
+ String DEFAULT_PRIMARY_KEY_UNICODE_CLEAN = "false";
+
+ }
+
+ /**
+ * This is a global option since it was always <code>false</code> for
+ * historical stores.
+ *
+ * @see Options#PRIMARY_KEY_UNICODE_CLEAN
+ */
+ final static transient boolean primaryKeyUnicodeClean = Boolean
+ .valueOf(System.getProperty(
+ SparseRowStore.Options.PRIMARY_KEY_UNICODE_CLEAN,
+ SparseRowStore.Options.DEFAULT_PRIMARY_KEY_UNICODE_CLEAN));
+
}
Deleted: branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractUnicodeKeyBuilderTestCase.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractUnicodeKeyBuilderTestCase.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractUnicodeKeyBuilderTestCase.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -1,183 +0,0 @@
-/*
-
-Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-*/
-/*
- * Created on Nov 29, 2007
- */
-
-package com.bigdata.btree;
-
-import java.util.Locale;
-import java.util.Properties;
-
-import junit.framework.TestCase2;
-
-import com.bigdata.btree.keys.DefaultKeyBuilderFactory;
-import com.bigdata.btree.keys.IKeyBuilder;
-import com.bigdata.btree.keys.KeyBuilder;
-import com.bigdata.btree.keys.StrengthEnum;
-import com.bigdata.btree.keys.KeyBuilder.Options;
-
-/**
- * Base class for the test suites that examine support for Unicode sort keys in
- * {@link KeyBuilder}.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- */
-abstract public class AbstractUnicodeKeyBuilderTestCase extends TestCase2 {
-
- /**
- *
- */
- public AbstractUnicodeKeyBuilderTestCase() {
- }
-
- /**
- * @param arg0
- */
- public AbstractUnicodeKeyBuilderTestCase(String arg0) {
- super(arg0);
- }
-
- /**
- * Test ability to encode unicode data into a variable length byte[] that
- * allows direct byte-by-byte comparisons which maintain the local-specific
- * sort order of the original strings.
- */
- public void test_keyBuilder_unicode_string_key_us_primary() {
-
- /*
- * Setup an instance for US English with strength := PRIMARY.
- */
-
- Properties properties = getProperties();
-
- properties.setProperty(Options.USER_LANGUAGE, Locale.US.getLanguage());
-
- properties.setProperty(Options.USER_COUNTRY, Locale.US.getCountry());
-
- properties.setProperty(Options.STRENGTH, StrengthEnum.Primary
- .toString());
-
- final DefaultKeyBuilderFactory factory = new DefaultKeyBuilderFactory(
- properties);
-
- assertEquals(Locale.US.getLanguage(), factory.getLocale().getLanguage());
-
- assertEquals(Locale.US.getCountry(), factory.getLocale().getCountry());
-
- assertEquals(StrengthEnum.Primary, factory.getStrength());
-
- final IKeyBuilder keyBuilder = factory.getKeyBuilder();
-
-// // verify assumption under that configuration.
-// {
-// RuleBasedCollator usCollator = (RuleBasedCollator) Collator
-// .getInstance(Locale.US);
-//
-// usCollator.setStrength(Collator.PRIMARY);
-//
-// assertEquals(0, usCollator.compare("abc", "ABC"));
-// }
-
- byte[] key1 = keyBuilder.reset().append("abc").getKey();
- byte[] key2 = keyBuilder.reset().append("ABC").getKey();
- byte[] key3 = keyBuilder.reset().append("Abc").getKey();
-
- System.err.println("abc: "+BytesUtil.toString(key1));
- System.err.println("ABC: "+BytesUtil.toString(key2));
- System.err.println("Abc: "+BytesUtil.toString(key3));
-
- // all are equal using PRIMARY strength.
- assertEquals(0,BytesUtil.compareBytes(key1, key2));
- assertEquals(0,BytesUtil.compareBytes(key2, key3));
-
- }
-
- public void test_keyBuilder_unicode_string_key_us_identical() {
-
- /*
- * Setup an instance for US English with strength := IDENTICAL.
- */
-
- Properties properties = new Properties();
-
- properties.setProperty(Options.USER_LANGUAGE, Locale.US.getLanguage());
-
- properties.setProperty(Options.USER_COUNTRY, Locale.US.getCountry());
-
- properties.setProperty(Options.STRENGTH, ""+StrengthEnum.Identical);
-
- IKeyBuilder keyBuilder = KeyBuilder.newUnicodeInstance(properties);
-
-// // verify assumption under that configuration.
-// {
-// RuleBasedCollator usCollator = (RuleBasedCollator) Collator
-// .getInstance(Locale.US);
-//
-// usCollator.setStrength(Collator.IDENTICAL);
-//
-// assertNotSame(0, usCollator.compare("abc", "ABC"));
-// }
-
- // IKeyBuilder keyBuilder = new UnicodeKeyBuilder(usCollator,1000);
-
- byte[] key1 = keyBuilder.reset().append("abc").getKey();
- byte[] key2 = keyBuilder.reset().append("ABC").getKey();
- byte[] key3 = keyBuilder.reset().append("Abc").getKey();
-
- System.err.println("abc: "+BytesUtil.toString(key1));
- System.err.println("ABC: "+BytesUtil.toString(key2));
- System.err.println("Abc: "+BytesUtil.toString(key3));
-
- // verify ordering for IDENTICAL comparison.
- assertTrue(BytesUtil.compareBytes(key1, key2)<0);
- assertTrue(BytesUtil.compareBytes(key2, key3)>0);
-
- }
-
- /**
- * Test verifies that the trailing <code>nul</code> byte is not part of
- * the key when a unicode string is appended to an {@link IKeyBuilder}.
- * <p>
- * Note: The trailing <code>nul</code> byte is appended by the ICU library
- * in order to have compatibility with their C library, but it is not of
- * interest for Java processing. However, note that a <code>nul</code>
- * byte MAY be used to separate components of a complex key.
- */
- public void test_keyBuilder_unicode_String_noTrailingNul() {
-
- IKeyBuilder keyBuilder = KeyBuilder.newUnicodeInstance(getProperties());
-
- keyBuilder.append("Hello World!");
-
- byte[] key = keyBuilder.getKey();
-
- assertNotSame("Not expecting a trailing nul byte.", (byte) 0,
- key[key.length - 1]);
-
- }
-
-}
Copied: branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java (from rev 3269, trunk/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java)
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java (rev 0)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -0,0 +1,342 @@
+/*
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+*/
+/*
+ * Created on Nov 29, 2007
+ */
+
+package com.bigdata.btree.keys;
+
+import java.util.Arrays;
+import java.util.Locale;
+import java.util.Properties;
+
+import junit.framework.TestCase2;
+
+import com.bigdata.btree.BytesUtil;
+import com.bigdata.btree.keys.DefaultKeyBuilderFactory;
+import com.bigdata.btree.keys.IKeyBuilder;
+import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.btree.keys.StrengthEnum;
+import com.bigdata.btree.keys.KeyBuilder.Options;
+
+/**
+ * Base class for the test suites that examine support for Unicode sort keys in
+ * {@link KeyBuilder}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+abstract public class AbstractUnicodeKeyBuilderTestCase extends TestCase2 {
+
+ /**
+ *
+ */
+ public AbstractUnicodeKeyBuilderTestCase() {
+ }
+
+ /**
+ * @param arg0
+ */
+ public AbstractUnicodeKeyBuilderTestCase(String arg0) {
+ super(arg0);
+ }
+
+ /**
+ * Test ability to encode unicode data into a variable length byte[] that
+ * allows direct byte-by-byte comparisons which maintain the local-specific
+ * sort order of the original strings.
+ */
+ public void test_keyBuilder_unicode_string_key_us_primary() {
+
+ /*
+ * Setup an instance for US English with strength := PRIMARY.
+ */
+
+ final Properties properties = getProperties();
+
+ properties.setProperty(Options.USER_LANGUAGE, Locale.US.getLanguage());
+
+ properties.setProperty(Options.USER_COUNTRY, Locale.US.getCountry());
+
+ properties.setProperty(Options.STRENGTH, StrengthEnum.Primary
+ .toString());
+
+ final DefaultKeyBuilderFactory factory = new DefaultKeyBuilderFactory(
+ properties);
+
+ assertEquals(Locale.US.getLanguage(), factory.getLocale().getLanguage());
+
+ assertEquals(Locale.US.getCountry(), factory.getLocale().getCountry());
+
+ assertEquals(StrengthEnum.Primary, factory.getStrength());
+
+ final IKeyBuilder keyBuilder = factory.getKeyBuilder();
+
+// // verify assumption under that configuration.
+// {
+// RuleBasedCollator usCollator = (RuleBasedCollator) Collator
+// .getInstance(Locale.US);
+//
+// usCollator.setStrength(Collator.PRIMARY);
+//
+// assertEquals(0, usCollator.compare("abc", "ABC"));
+// }
+
+ byte[] key1 = keyBuilder.reset().append("abc").getKey();
+ byte[] key2 = keyBuilder.reset().append("ABC").getKey();
+ byte[] key3 = keyBuilder.reset().append("Abc").getKey();
+
+ System.err.println("abc: "+BytesUtil.toString(key1));
+ System.err.println("ABC: "+BytesUtil.toString(key2));
+ System.err.println("Abc: "+BytesUtil.toString(key3));
+
+ // all are equal using PRIMARY strength.
+ assertEquals(0,BytesUtil.compareBytes(key1, key2));
+ assertEquals(0,BytesUtil.compareBytes(key2, key3));
+
+ }
+
+ public void test_keyBuilder_unicode_string_key_us_identical() {
+
+ /*
+ * Setup an instance for US English with strength := IDENTICAL.
+ */
+
+ final Properties properties = new Properties();
+
+ properties.setProperty(Options.USER_LANGUAGE, Locale.US.getLanguage());
+
+ properties.setProperty(Options.USER_COUNTRY, Locale.US.getCountry());
+
+ properties.setProperty(Options.STRENGTH, ""+StrengthEnum.Identical);
+
+ final IKeyBuilder keyBuilder = KeyBuilder.newUnicodeInstance(properties);
+
+// // verify assumption under that configuration.
+// {
+// RuleBasedCollator usCollator = (RuleBasedCollator) Collator
+// .getInstance(Locale.US);
+//
+// usCollator.setStrength(Collator.IDENTICAL);
+//
+// assertNotSame(0, usCollator.compare("abc", "ABC"));
+// }
+
+ // IKeyBuilder keyBuilder = new UnicodeKeyBuilder(usCollator,1000);
+
+ byte[] key1 = keyBuilder.reset().append("abc").getKey();
+ byte[] key2 = keyBuilder.reset().append("ABC").getKey();
+ byte[] key3 = keyBuilder.reset().append("Abc").getKey();
+
+ System.err.println("abc: "+BytesUtil.toString(key1));
+ System.err.println("ABC: "+BytesUtil.toString(key2));
+ System.err.println("Abc: "+BytesUtil.toString(key3));
+
+ // verify ordering for IDENTICAL comparison.
+ assertTrue(BytesUtil.compareBytes(key1, key2)<0);
+ assertTrue(BytesUtil.compareBytes(key2, key3)>0);
+
+ }
+
+ /**
+ * Test verifies that the trailing <code>nul</code> byte is not part of
+ * the key when a unicode string is appended to an {@link IKeyBuilder}.
+ * <p>
+ * Note: The trailing <code>nul</code> byte is appended by the ICU library
+ * in order to have compatibility with their C library, but it is not of
+ * interest for Java processing. However, note that a <code>nul</code>
+ * byte MAY be used to separate components of a complex key.
+ */
+ public void test_keyBuilder_unicode_String_noTrailingNul() {
+
+ final IKeyBuilder keyBuilder = KeyBuilder.newUnicodeInstance(getProperties());
+
+ keyBuilder.append("Hello World!");
+
+ final byte[] key = keyBuilder.getKey();
+
+ assertNotSame("Not expecting a trailing nul byte.", (byte) 0,
+ key[key.length - 1]);
+
+ }
+
+ /**
+ * Test verifies that the <code>nul</code> byte is not part of the key when
+ * a unicode string is appended to an {@link IKeyBuilder}.
+ * <p>
+ * Note: The {@link SparseRowStore} assumes that Unicode sort keys do not
+ * contain embedded <code>nul</code>s.
+ */
+ public void test_keyBuilder_unicode_String_noEmbeddedNuls() {
+
+ final IKeyBuilder keyBuilder = KeyBuilder
+ .newUnicodeInstance(getProperties());
+
+ keyBuilder.append("Hello World!");
+
+ final byte[] key = keyBuilder.getKey();
+
+ for (int i = 0; i < key.length; i++) {
+ if (key[i] == 0)
+ fail("Embedded nuls: key=" + BytesUtil.toString(key));
+ }
+
+ }
+
+ /**
+ * Test of the ability to normalize trailing pad characters.
+ */
+ public void test_keyBuilder_normalizeTrailingPadCharacters() {
+
+ final KeyBuilder keyBuilder = (KeyBuilder) KeyBuilder
+ .newUnicodeInstance(getProperties());
+
+ assertEquals(//
+ keyBuilder.normalizeText(""),//
+ keyBuilder.normalizeText(" ")//
+ );
+ assertEquals(//
+ keyBuilder.normalizeText(""),//
+ keyBuilder.normalizeText(" ")//
+ );
+ assertEquals(//
+ keyBuilder.normalizeText(""),//
+ keyBuilder.normalizeText(" ")//
+ );
+ assertEquals(//
+ keyBuilder.normalizeText(" "),//
+ keyBuilder.normalizeText(" ")//
+ );
+ assertEquals(//
+ keyBuilder.normalizeText("abc"),//
+ keyBuilder.normalizeText("abc ")//
+ );
+ assertEquals(//
+ keyBuilder.normalizeText(" abc"),//
+ keyBuilder.normalizeText(" abc ")//
+ );
+ assertNotSame(//
+ keyBuilder.normalizeText("abc"),//
+ keyBuilder.normalizeText(" abc ")//
+ );
+
+ }
+
+ /**
+ * Test verifies that very long strings are truncated.
+ *
+ * @todo verify that trailing whitespace is removed after truncation rather
+ * than before truncation.
+ */
+ public void test_keyBuilder_normalizeTruncatesVeryLongStrings() {
+
+ final KeyBuilder keyBuilder = (KeyBuilder)KeyBuilder.newUnicodeInstance(getProperties());
+
+ final String text = TestKeyBuilder.getMaximumLengthText();
+
+ assertEquals(//
+ keyBuilder.normalizeText(text),//
+ keyBuilder.normalizeText(text+"abc")//
+ );
+
+ }
+
+ /**
+ * Test verifies the order among unicode sort keys, including verifying that
+ * the pad byte causes a prefix such as "bro" to sort before a term which
+ * extends that prefix, such as "brown".
+ */
+ public void test_keyBuilder_unicode_order() {
+
+ final KeyBuilder keyBuilder = (KeyBuilder) KeyBuilder.newUnicodeInstance(getProperties());
+
+ final KVO<String>[] a = new KVO[] {
+
+ new KVO<String>(keyBuilder.asSortKey("bro"),null,"bro"),
+ new KVO<String>(keyBuilder.asSortKey("brown"),null,"brown"),
+ new KVO<String>(keyBuilder.asSortKey("bre"),null,"bre"),
+ new KVO<String>(keyBuilder.asSortKey("break"),null,"break"),
+
+ };
+
+ // sort by the assigned sort keys.
+ Arrays.sort(a);
+
+ /*
+ * verify that "bre(ak)" is before "bro(wn)" and that "bre" is before
+ * "break" and "bro" is before "brown".
+ */
+ assertEquals("bre", a[0].obj);
+ assertEquals("break", a[1].obj);
+ assertEquals("bro", a[2].obj);
+ assertEquals("brown", a[3].obj);
+
+ }
+
+ /**
+ * <p>
+ * Test that lexiographic order is maintain when a variable length Unicode
+ * field is followed by another field. This test works by comparing the
+ * original multi-field key with the multi-field key formed from the
+ * successor of the Unicode field followed by the other field:
+ * </p>
+ *
+ * <pre>
+ *
+ * [text][nextValue] LT [successor(text)][nextValue]
+ *
+ * </pre>
+ */
+ public void test_keyBuilder_multiField_unicode() {
+
+ final KeyBuilder keyBuilder = (KeyBuilder) KeyBuilder
+ .newUnicodeInstance(getProperties());
+
+ TestKeyBuilder.doMultiFieldTests(true/* unicode */, keyBuilder);
+
+ /*
+ * Now test some strings that contain code points outside of the 8-bit
+ * range.
+ */
+
+// final KeyBuilder keyBuilder = (KeyBuilder) KeyBuilder
+// .newUnicodeInstance();
+
+ final boolean unicode = true;
+ {
+
+ // Note: This is "Japanese" in kanji.
+ String text = "\u65E5\u672C\u8A9E / \u306B\u307B\u3093\u3054";
+
+ TestKeyBuilder.doMultiFieldTest(keyBuilder, unicode, text, (byte) 0);
+ TestKeyBuilder.doMultiFieldTest(keyBuilder, unicode, text, (byte) 1);
+ TestKeyBuilder.doMultiFieldTest(keyBuilder, unicode, text, (byte) -1);
+ TestKeyBuilder.doMultiFieldTest(keyBuilder, unicode, text, Byte.MIN_VALUE);
+ TestKeyBuilder.doMultiFieldTest(keyBuilder, unicode, text, Byte.MAX_VALUE);
+ }
+
+ }
+
+}
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestICUUnicodeKeyBuilder.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestICUUnicodeKeyBuilder.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestICUUnicodeKeyBuilder.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -31,7 +31,6 @@
import java.util.Locale;
import java.util.Properties;
-import com.bigdata.btree.AbstractUnicodeKeyBuilderTestCase;
import com.bigdata.btree.BytesUtil;
import com.bigdata.btree.keys.KeyBuilder.Options;
import com.ibm.icu.text.Collator;
@@ -39,15 +38,16 @@
/**
* Tests for Unicode support in {@link KeyBuilder}.
*
- * @todo test w/ and w/o the ICU integration (can be choosen at run time). Note
- * that some tests are specific to the ICU libraries at this time.
+ * @todo test w/ and w/o the ICU integration (can be chosen at run time via
+ * concrete subclasses). Note that some tests are specific to the ICU
+ * libraries at this time.
*
* @todo write performance test for encoding strings, possibly in the context of
* parsed rdf data, and see if there are any easy wins in how the encoding
* to a sort key is handled or in alignment of the apis.
*
* @todo compare performance of the ICU and JDK libraries in some application
- * contexts. compare performance of the JNI ICU library as well.
+ * contexts. compare performance of the JNI ICU library as well.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
@@ -69,7 +69,7 @@
public Properties getProperties() {
- Properties properties = new Properties(super.getProperties());
+ final Properties properties = new Properties(super.getProperties());
properties.setProperty(Options.COLLATOR,CollatorEnum.ICU.toString());
@@ -79,11 +79,12 @@
public void test_correctCollator() {
- Properties properties = getProperties();
-
- log.info("properties="+properties);
-
- KeyBuilder keyBuilder = (KeyBuilder) KeyBuilder
+ final Properties properties = getProperties();
+
+ if (log.isInfoEnabled())
+ log.info("properties=" + properties);
+
+ final KeyBuilder keyBuilder = (KeyBuilder) KeyBuilder
.newUnicodeInstance(properties);
assertEquals(ICUSortKeyGenerator.class, keyBuilder
@@ -109,13 +110,13 @@
* Setup for US English.
*/
- Properties properties = new Properties();
+ final Properties properties = new Properties();
properties.setProperty(Options.USER_LANGUAGE, Locale.US.getLanguage());
properties.setProperty(Options.USER_COUNTRY, Locale.US.getCountry());
- int[] strengths = new int[] {
+ final int[] strengths = new int[] {
Collator.PRIMARY,
Collator.SECONDARY,
Collator.TERTIARY,
@@ -174,7 +175,7 @@
* @param collator
* The collator.
*
- * @return True iff the collector differenties between the string and its
+ * @return True iff the collector differentiates between the string and its
* successor (formed by appending a nul character) in its generated
* sort keys.
*/
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestJDKUnicodeKeyBuilder.java
===========...
[truncated message content] |
|
From: <mrp...@us...> - 2010-07-23 18:36:49
|
Revision: 3275
http://bigdata.svn.sourceforge.net/bigdata/?rev=3275&view=rev
Author: mrpersonick
Date: 2010-07-23 18:36:43 +0000 (Fri, 23 Jul 2010)
Log Message:
-----------
cleaning up, documenting
Modified Paths:
--------------
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ExtensionIV.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java 2010-07-23 18:04:32 UTC (rev 3274)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ColorsEnumExtension.java 2010-07-23 18:36:43 UTC (rev 3275)
@@ -37,10 +37,14 @@
public static final URI COLOR = new URIImpl(BD.NAMESPACE + "Color");
- private final BigdataURI color;
+ private BigdataURI color;
- public ColorsEnumExtension(final IDatatypeURIResolver resolver) {
+ public ColorsEnumExtension() {
+ }
+
+ public void resolveDatatype(final IDatatypeURIResolver resolver) {
+
this.color = resolver.resolve(COLOR);
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java 2010-07-23 18:04:32 UTC (rev 3274)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java 2010-07-23 18:36:43 UTC (rev 3275)
@@ -1,18 +1,31 @@
package com.bigdata.rdf.internal;
+/**
+ * Simple {@link IExtensionFactory} implementation that creates two
+ * {@link IExtension}s - the {@link EpochExtension} and the
+ * {@link ColorsEnumExtension}.
+ */
public class DefaultExtensionFactory implements IExtensionFactory {
private final IExtension[] extensions;
- public DefaultExtensionFactory(final IDatatypeURIResolver resolver) {
+ public DefaultExtensionFactory() {
extensions = new IExtension[] {
- new EpochExtension(resolver),
- new ColorsEnumExtension(resolver)
+ new EpochExtension(),
+ new ColorsEnumExtension()
};
}
+ public void resolveDatatypes(final IDatatypeURIResolver resolver) {
+
+ for (IExtension extension : extensions) {
+ extension.resolveDatatype(resolver);
+ }
+
+ }
+
public IExtension[] getExtensions() {
return extensions;
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java 2010-07-23 18:04:32 UTC (rev 3274)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/EpochExtension.java 2010-07-23 18:36:43 UTC (rev 3275)
@@ -38,10 +38,14 @@
public static final URI EPOCH = new URIImpl(BD.NAMESPACE + "Epoch");
- private final BigdataURI epoch;
+ private BigdataURI epoch;
- public EpochExtension(final IDatatypeURIResolver resolver) {
+ public EpochExtension() {
+ }
+
+ public void resolveDatatype(final IDatatypeURIResolver resolver) {
+
this.epoch = resolver.resolve(EPOCH);
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ExtensionIV.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ExtensionIV.java 2010-07-23 18:04:32 UTC (rev 3274)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ExtensionIV.java 2010-07-23 18:36:43 UTC (rev 3275)
@@ -78,11 +78,15 @@
return delegate.byteLength() + Bytes.SIZEOF_LONG;
}
+ /**
+ * Defer to the {@link ILexiconConfiguration} which has specific knowledge
+ * of how to generate an RDF value from this general purpose extension IV.
+ */
@SuppressWarnings("unchecked")
public V asValue(final BigdataValueFactory f,
final ILexiconConfiguration config)
throws UnsupportedOperationException {
- return (V) config.getExtension(datatype).asValue(this, f);
+ return (V) config.asValue(this, f);
}
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java 2010-07-23 18:04:32 UTC (rev 3274)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java 2010-07-23 18:36:43 UTC (rev 3275)
@@ -28,13 +28,55 @@
import com.bigdata.rdf.model.BigdataURI;
import com.bigdata.rdf.model.BigdataValueFactory;
-
+/**
+ * IExtensions are responsible for roundtripping between an RDF value and an
+ * {@link ExtensionIV} for a particular datatype. Because of how
+ * {@link ExtensionIV}s are encoded and decoded, the IExtension will need to
+ * have on hand the {@link TermId} for its datatype. This is accomplished
+ * via the {@link IDatatypeURIResolver} - the IExtension will give the resolver
+ * the datatype URI it needs resolved and the resolver will lookup (or create)
+ * the {@link TermId}.
+ */
public interface IExtension {
+ /**
+ * This will be called very early in the IExtension lifecycle so that the
+ * {@link TermId} for the datatype URI will be on hand when needed.
+ *
+ * @param resolver
+ * the datatype URI resolver
+ */
+ void resolveDatatype(final IDatatypeURIResolver resolver);
+
+ /**
+ * Return the fully resolved datatype in the form of a {@link BigdataURI}
+ * with the {@link TermId} already set.
+ *
+ * @return
+ * the datatype
+ */
BigdataURI getDatatype();
+ /**
+ * Create an {@link ExtensionIV} from an RDF value.
+ *
+ * @param value
+ * the RDF value
+ * @return
+ * the extension IV
+ */
ExtensionIV createIV(final Value value);
+ /**
+ * Create an RDF value from an {@link ExtensionIV}.
+ *
+ * @param iv
+ * the extension IV
+ * @param vf
+ * the bigdata value factory
+ * @return
+ * the RDF value
+ */
Value asValue(final ExtensionIV iv, final BigdataValueFactory vf);
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java 2010-07-23 18:04:32 UTC (rev 3274)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java 2010-07-23 18:36:43 UTC (rev 3275)
@@ -24,8 +24,29 @@
package com.bigdata.rdf.internal;
+/**
+ * IExtensionFactories are responsible for enumerating what extensions are
+ * supported for a particular database configuration. Bigdata comes packaged
+ * with a {@link DefaultExtensionFactory} that supplies two starter extensions -
+ * the {@link EpochExtension} (for representing time since the epoch as a long
+ * integer) and the {@link ColorsEnumExtension} (a sample extension for how to
+ * represent an enumeration via inline literals).
+ */
public interface IExtensionFactory {
+ /**
+ * This will be called very early in the IExtensionFactory lifecycle so that
+ * the {@link TermId}s for the {@link IExtension}'s datatype URIs will be on
+ * hand when needed.
+ *
+ * @param resolver
+ * the datatype URI resolver
+ */
+ void resolveDatatypes(final IDatatypeURIResolver resolver);
+
+ /**
+ * Return the supported extensions.
+ */
IExtension[] getExtensions();
}
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java 2010-07-23 18:04:32 UTC (rev 3274)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java 2010-07-23 18:36:43 UTC (rev 3275)
@@ -26,8 +26,9 @@
import org.openrdf.model.URI;
import org.openrdf.model.Value;
+import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.model.BigdataValueFactory;
-
/**
* Configuration determines which RDF Values are inlined into the statement
* indices rather than being assigned term identifiers by the lexicon.
@@ -35,16 +36,33 @@
public interface ILexiconConfiguration {
/**
- * <code>true</code> if the data model should use the legacy encoding
- * for terms - all long term identifiers without byte flags and no inline
- * values. This is for backward compatibility.
- public boolean isLegacyEncoding();
+ * Create an inline {@link IV} for the supplied RDF value if inlining is
+ * supported for the supplied RDF value.
+ * <p>
+ * If the supplied RDF value is a {@link BigdataValue} then the {@link IV}
+ * will be set as a side-effect.
+ *
+ * @param value
+ * the RDF value
+ * @return
+ * the inline {@link IV}
*/
+ IV createInlineIV(final Value value);
- IV createIV(final Value value);
+ /**
+ * Create an RDF value from an {@link ExtensionIV}. Looks through an
+ * internal catalog of {@link IExtension}s to find one that knows how to
+ * handle the extension datatype from the supplied {@link ExtensionIV}.
+ *
+ * @param iv
+ * the extension IV
+ * @param vf
+ * the bigdata value factory
+ * @return
+ * the RDF value
+ */
+ Value asValue(final ExtensionIV iv, final BigdataValueFactory vf);
- IExtension getExtension(final TermId datatype);
-
/**
* <code>true</code> iff the <code>vte</code> and <code>dte</code>
* should be inlined.
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java 2010-07-23 18:04:32 UTC (rev 3274)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java 2010-07-23 18:36:43 UTC (rev 3275)
@@ -37,6 +37,7 @@
import org.openrdf.model.datatypes.XMLDatatypeUtil;
import com.bigdata.rdf.model.BigdataURI;
import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.model.BigdataValueFactory;
/**
@@ -75,11 +76,12 @@
}
- public IExtension getExtension(final TermId datatype) {
- return termIds.get(datatype);
+ public Value asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
+ final TermId datatype = iv.getExtensionDatatype();
+ return termIds.get(datatype).asValue(iv, vf);
}
- public IV createIV(final Value value) {
+ public IV createInlineIV(final Value value) {
// we know right away we can't handle URIs
if (value instanceof URI)
Modified: branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
===================================================================
--- branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-07-23 18:04:32 UTC (rev 3274)
+++ branches/LEXICON_REFACTOR_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-07-23 18:36:43 UTC (rev 3275)
@@ -2318,7 +2318,7 @@
*/
private IV getInlineIV(Value value) {
- return getLexiconConfiguration().createIV(value);
+ return getLexiconConfiguration().createInlineIV(value);
}
@@ -2636,24 +2636,22 @@
try {
- final Class<IExtensionFactory> xfc = determineExtensionFactoryClass();
- final Constructor<IExtensionFactory> ctor = xfc.getConstructor(
- IDatatypeURIResolver.class);
- final IExtensionFactory xFactory = ctor.newInstance(
- (IDatatypeURIResolver) this);
+ final Class<IExtensionFactory> xfc =
+ determineExtensionFactoryClass();
+ final IExtensionFactory xFactory = xfc.newInstance();
+ /*
+ * Allow the extensions to resolve their datatype URIs into
+ * term identifiers.
+ */
+ xFactory.resolveDatatypes(this);
+
lexiconConfiguration = new LexiconConfiguration(
inlineLiterals, inlineBNodes, xFactory);
} catch (InstantiationException e) {
throw new IllegalArgumentException(
AbstractTripleStore.Options.EXTENSION_FACTORY_CLASS, e);
- } catch (NoSuchMethodException e) {
- throw new IllegalArgumentException(
- AbstractTripleStore.Options.EXTENSION_FACTORY_CLASS, e);
- } catch (InvocationTargetException e) {
- throw new IllegalArgumentException(
- AbstractTripleStore.Options.EXTENSION_FACTORY_CLASS, e);
} catch (IllegalAccessException e) {
throw new IllegalArgumentException(
AbstractTripleStore.Options.EXTENSION_FACTORY_CLASS, e);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
4 messages has been excluded from this view by a project administrator.
