Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[4322] branches/QUADS_QUERY_BRANCH/bigdata/src
|
From: <tho...@us...> - 2011-03-21 16:40:20
|
Revision: 4322
http://bigdata.svn.sourceforge.net/bigdata/?rev=4322&view=rev
Author: thompsonbry
Date: 2011-03-21 16:40:13 +0000 (Mon, 21 Mar 2011)
Log Message:
-----------
Introduced an ICUVersionRecord and runtime checking of the deployed ICU version based on self-reporting via ICU's VersionInfo class. The ICUVersionRecord's address is stored in the ICommitRecord. Old stores will automatically introduce this record on their next commit. New stores will write this record when they are first created. Update to new versions of ICU will be refused based on an incompatible value for this record unless Options#UPDATE_ICUVERSION is forced to "true".
See https://sourceforge.net/apps/trac/bigdata/ticket/193
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/Options.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestICUPortabilityBug.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestDiskJournal.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUVersionRecord.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestICUVersionRecord.java
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUVersionRecord.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUVersionRecord.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUVersionRecord.java 2011-03-21 16:40:13 UTC (rev 4322)
@@ -0,0 +1,251 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Mar 21, 2011
+ */
+
+package com.bigdata.btree.keys;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+
+import com.bigdata.journal.Name2Addr;
+import com.ibm.icu.util.VersionInfo;
+
+/**
+ * Persistent record in which we store the version metadata for the ICU
+ * dependency in use when the journal was created. bigdata uses Unicode sort
+ * keys for various indices, including {@link Name2Addr}. A change in the ICU
+ * version can result in sort keys which are NOT compatible. Binary
+ * compatibility for Unicode sort keys is an absolute requirement for bigdata.
+ * The purpose of this persistence capable data record is to note the version of
+ * ICU against which bigdata was linked with the associated binary store file
+ * was created.
+ * <p>
+ * Note: This can result in data which apparently becomes "lost", such as this
+ * <a href="http://sourceforge.net/apps/trac/bigdata/ticket/193>trac issue</a>.
+ * The underlying problem was substituting a newer version of ICU for the one
+ * included in the bigdata distribution. Such errors are now caught by detecting
+ * a change in the ICU runtime environment.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public class ICUVersionRecord implements Externalizable {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ private VersionInfo icuVersion;
+ private VersionInfo ucolRuntimeVersion;
+ private VersionInfo ucolBuilderVersion;
+ private VersionInfo ucolTailoringsVersion;
+
+ /**
+ * The ICU software version number.
+ *
+ * @see VersionInfo#ICU_VERSION
+ */
+ public VersionInfo getICUVersion() {
+ return icuVersion;
+ }
+
+ /**
+ * If this version number changes, then the sort keys for the same Unicode
+ * string could be different.
+ *
+ * @see VersionInfo#UCOL_RUNTIME_VERSION
+ */
+ public VersionInfo getUColRuntimeVersion() {
+ return ucolRuntimeVersion;
+ }
+
+ /**
+ * If this version number changes, then the same tailoring might result in
+ * assigning different collation elements to code points (which could break
+ * binary compatibility on sort keys).
+ *
+ * @see VersionInfo#UCOL_BUILDER_VERSION
+ */
+ public VersionInfo getUColBuilderVersion() {
+ return ucolBuilderVersion;
+ }
+
+ /**
+ * The version of the collation tailorings.
+ *
+ * @see VersionInfo#UCOL_TAILORINGS_VERSION
+ */
+ public VersionInfo getUColTailoringsVersion() {
+ return ucolTailoringsVersion;
+ }
+
+ /**
+ * Factory returns a record reporting on the ICU dependency as currently
+ * linked with the code base.
+ */
+ public static ICUVersionRecord newInstance() {
+
+ final ICUVersionRecord r = new ICUVersionRecord(//
+ VersionInfo.ICU_VERSION,//
+ VersionInfo.UCOL_RUNTIME_VERSION,//
+ VersionInfo.UCOL_BUILDER_VERSION,//
+ VersionInfo.UCOL_TAILORINGS_VERSION//
+ );
+
+ return r;
+
+ }
+
+ private ICUVersionRecord(//
+ VersionInfo icuVersion,//
+ VersionInfo ucolRuntimeVesion,//
+ VersionInfo ucolBuilderVersion,//
+ VersionInfo ucolTailoringsVersion//
+ ) {
+
+ this.icuVersion = icuVersion;
+
+ this.ucolRuntimeVersion = ucolRuntimeVesion;
+
+ this.ucolBuilderVersion = ucolBuilderVersion;
+
+ this.ucolTailoringsVersion = ucolTailoringsVersion;
+
+ }
+
+ /**
+ * De-serialization contructor <strong>only</strong>.
+ */
+ public ICUVersionRecord() {
+
+ }
+
+
+ /** The initial version of this data record. */
+ private static final transient int VERSION0 = 0;
+
+ private static final transient int CURRENT_VERSION = VERSION0;
+
+ public void readExternal(ObjectInput in) throws IOException,
+ ClassNotFoundException {
+
+ final int version = in.readInt();
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new IOException("Unknown version: " + version);
+ }
+
+ icuVersion = readVersionInfo(in);
+ ucolRuntimeVersion = readVersionInfo(in);
+ ucolBuilderVersion = readVersionInfo(in);
+ ucolTailoringsVersion = readVersionInfo(in);
+
+ }
+
+ public void writeExternal(ObjectOutput out) throws IOException {
+
+ out.writeInt(CURRENT_VERSION);
+ writeVersionInfo(icuVersion, out);
+ writeVersionInfo(ucolRuntimeVersion, out);
+ writeVersionInfo(ucolBuilderVersion, out);
+ writeVersionInfo(ucolTailoringsVersion, out);
+
+ }
+
+ private VersionInfo readVersionInfo(final ObjectInput in) throws IOException {
+
+ final int major = in.readInt();
+ final int minor = in.readInt();
+ final int milli = in.readInt();
+ final int micro = in.readInt();
+
+ return VersionInfo.getInstance(major, minor, milli, micro);
+
+ }
+
+ private void writeVersionInfo(final VersionInfo v, final ObjectOutput out)
+ throws IOException {
+
+ out.writeInt(v.getMajor());
+ out.writeInt(v.getMinor());
+ out.writeInt(v.getMicro());
+ out.writeInt(v.getMilli());
+
+ }
+
+ /**
+ * A human readable representation of the data record.
+ */
+ public String toString() {
+ final StringBuilder sb = new StringBuilder();
+ sb.append(getClass().getName());
+ sb.append("{icuVersion=" + icuVersion);
+ sb.append(",ucolRuntimeVersion=" + ucolRuntimeVersion);
+ sb.append(",ucolBuilderVersion=" + ucolBuilderVersion);
+ sb.append(",ucolTailoringsVersion=" + ucolTailoringsVersion);
+ sb.append("}");
+ return sb.toString();
+ }
+
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ public boolean equals(final Object o) {
+ if (this == o)
+ return true;
+ if (!(o instanceof ICUVersionRecord))
+ return false;
+ final ICUVersionRecord r = (ICUVersionRecord) o;
+ if (!icuVersion.equals(r.icuVersion))
+ return false;
+ if (!ucolRuntimeVersion.equals(r.ucolRuntimeVersion))
+ return false;
+ if (!ucolBuilderVersion.equals(r.ucolBuilderVersion))
+ return false;
+ if (!ucolTailoringsVersion.equals(r.ucolTailoringsVersion))
+ return false;
+ return true;
+ }
+
+ /**
+ * Writes out the {@link ICUVersionRecord} for the current classpath.
+ *
+ * @param args
+ * Ignored.
+ */
+ static public void main(String[] args) {
+
+ System.out.println(ICUVersionRecord.newInstance().toString());
+
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUVersionRecord.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2011-03-21 16:39:10 UTC (rev 4321)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2011-03-21 16:40:13 UTC (rev 4322)
@@ -60,6 +60,7 @@
import com.bigdata.btree.IIndex;
import com.bigdata.btree.IndexMetadata;
import com.bigdata.btree.ReadOnlyIndex;
+import com.bigdata.btree.keys.ICUVersionRecord;
import com.bigdata.cache.ConcurrentWeakValueCache;
import com.bigdata.cache.ConcurrentWeakValueCacheWithTimeout;
import com.bigdata.cache.HardReferenceQueue;
@@ -76,6 +77,7 @@
import com.bigdata.ha.QuorumService;
import com.bigdata.io.IDataRecord;
import com.bigdata.io.IDataRecordAccess;
+import com.bigdata.io.SerializerUtil;
import com.bigdata.journal.Name2Addr.Entry;
import com.bigdata.journal.ha.HAWriteMessage;
import com.bigdata.mdi.IResourceMetadata;
@@ -174,18 +176,25 @@
*/
public static transient final int ROOT_NAME2ADDR = 0;
- /**
- * The index of the address where the root block copy from the previous
- * commit is stored
- */
+ /**
+ * The index of the root address where the root block copy from the previous
+ * commit is stored.
+ */
public static transient final int PREV_ROOTBLOCK = 1;
/**
- * The index of the address of the delete blocks associated with
- * this transaction
+ * The index of the root address of the delete blocks associated with
+ * this transaction.
*/
public static transient final int DELETEBLOCK = 2;
+ /**
+ * The index of the root address containing the {@link ICUVersionRecord}.
+ * That record specifies the ICU version metadata which was in force when
+ * the journal was created.
+ */
+ public static transient final int ROOT_ICUVERSION = 3;
+
/**
* A clone of the properties used to initialize the {@link Journal}.
*/
@@ -291,6 +300,11 @@
private volatile CommitRecordIndex _commitRecordIndex;
/**
+ * The {@link ICUVersionRecord} iff known.
+ */
+ private volatile ICUVersionRecord _icuVersionRecord;
+
+ /**
* The configured capacity for the {@link HardReferenceQueue} backing the
* index cache maintained by the "live" {@link Name2Addr} object.
*
@@ -1003,6 +1017,27 @@
// new or re-load commit record index from store via root block.
this._commitRecordIndex = _getCommitRecordIndex();
+ // new or re-load from the store.
+ this._icuVersionRecord = _getICUVersionRecord();
+
+ // verify the ICU version.
+ if (this._icuVersionRecord != null
+ && !ICUVersionRecord.newInstance().equals(
+ this._icuVersionRecord)) {
+
+ final boolean update = Boolean.valueOf(properties.getProperty(
+ Options.UPDATE_ICU_VERSION, "false"));
+
+ if (!update) {
+
+ throw new RuntimeException("ICUVersionChange: store="
+ + this._icuVersionRecord + ", runtime="
+ + ICUVersionRecord.newInstance());
+
+ }
+
+ }
+
// Give the store a chance to set any committers that it defines.
setupCommitters();
@@ -2081,6 +2116,9 @@
// clear reference and reload from the store.
_commitRecordIndex = _getCommitRecordIndex();
+ // clear reference and reload from the store.
+ _icuVersionRecord = _getICUVersionRecord();
+
// clear the array of committers.
_committers = new ICommitter[_committers.length];
@@ -2813,12 +2851,114 @@
if (_bufferStrategy instanceof RWStrategy)
setCommitter(DELETEBLOCK, new DeleteBlockCommitter((RWStrategy) _bufferStrategy));
+ /*
+ * Responsible for writing the ICUVersionRecord exactly once onto
+ * the backing store, e.g., when the store is created or when it is
+ * open with the "update" option specified for ICU.
+ */
+ setCommitter(ROOT_ICUVERSION, new ICUVersionCommitter());
-
}
}
+ /**
+ * Return the {@link ICUVersionRecord} from the current
+ * {@link ICommitRecord} -or- a new instance for the current runtime
+ * environment if the root address for {@link #ROOT_ICUVERSION} is
+ * {@link #NULL}.
+ */
+ private ICUVersionRecord _getICUVersionRecord() {
+
+ assert _fieldReadWriteLock.writeLock().isHeldByCurrentThread();
+
+ final long addr = getRootAddr(ROOT_ICUVERSION);
+
+ final ICUVersionRecord r;
+ if (addr == NULL) {
+ // New instance for the current runtime environment.
+ r = ICUVersionRecord.newInstance();
+ } else {
+ // Existing instance from the store.
+ r = (ICUVersionRecord) SerializerUtil.deserialize(read(addr));
+ }
+ return r;
+
+ }
+
+ /**
+ * Writes the {@link ICUVersionRecord} onto the store iff either (a) it does
+ * not exist; or (b) it exists, it differs from the last persistent record,
+ * and the update flag was specified.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan
+ * Thompson</a>
+ *
+ * @see Options#UPDATE_ICU_VERSION
+ */
+ private class ICUVersionCommitter implements ICommitter {
+
+ private boolean update;
+
+ private long lastAddr;
+
+ private ICUVersionCommitter() {
+
+ // the "update" option.
+ update = Boolean.valueOf(properties.getProperty(
+ Options.UPDATE_ICU_VERSION, "false"));
+
+ // lookup the address of the ICU version record (may be NULL).
+ lastAddr = getRootAddr(ROOT_ICUVERSION);
+
+ }
+
+ /**
+ * Commits a new {@link ICUVersionRecord} IF none is defined -OR- IF one
+ * is defined, it is a different version of ICU, and the update flag is
+ * set.
+ */
+ public long handleCommit(final long commitTime) {
+
+ if(!update && lastAddr != NULL) {
+
+ // Nothing changed.
+ return lastAddr;
+
+ }
+
+ /*
+ * Note: The Journal only validates the persistent ICU version
+ * record in its constructor. By the time the code reaches this
+ * point, it is either in agreement or will be written.
+ */
+
+ final ICUVersionRecord r = ICUVersionRecord.newInstance();
+
+ if (lastAddr == NULL || !(r.equals(_icuVersionRecord) && update)) {
+
+ if (_icuVersionRecord != null && update)
+ log.warn("Updating ICUVersion: old=" + _icuVersionRecord
+ + ", new=" + r);
+
+ // do not update next time.
+ update = false;
+
+ // write ICU version record onto the store.
+ lastAddr = write(ByteBuffer.wrap(SerializerUtil.serialize(r)));
+
+ // return address of the ICU version record.
+ return lastAddr;
+
+ }
+
+ // Nothing changed.
+ return lastAddr;
+
+ }
+
+ }
+
/*
* named indices.
*/
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/Options.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/Options.java 2011-03-21 16:39:10 UTC (rev 4321)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/Options.java 2011-03-21 16:40:13 UTC (rev 4322)
@@ -32,6 +32,7 @@
import com.bigdata.btree.Checkpoint;
import com.bigdata.btree.IndexSegment;
+import com.bigdata.btree.keys.ICUVersionRecord;
import com.bigdata.cache.HardReferenceQueue;
import com.bigdata.io.DirectBufferPool;
import com.bigdata.io.FileLockUtility;
@@ -351,6 +352,19 @@
String IGNORE_BAD_ROOT_BLOCK = AbstractJournal.class.getName()+".ignoreBadRootBlock";
/**
+ * <strong>WARNING - The use of this option is dangerous.</strong> This
+ * option may be used to update the {@link ICUVersionRecord} associated with
+ * the journal. ICU provides a Unicode sort key generation service for
+ * bigdata. Unicode sort keys are used in many indices, including the
+ * {@link Name2Addr} index. If the new ICU version produces Unicode sort
+ * keys which are not binary compatible with the Journal, then your data may
+ * become inaccessible since you will be unable to probe the
+ * {@link Name2Addr} index to locate named indices. The same problem can
+ * manifest with application indices which use Unicode sort keys.
+ */
+ String UPDATE_ICU_VERSION = AbstractJournal.class.getName()+".updateICUVersion";
+
+ /**
* An optional boolean property (default is {@value #DEFAULT_CREATE}). When
* <code>true</code> and the named file is not found, a new journal will be
* created. If the file exists but is empty, then a new journal will be
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestAll.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestAll.java 2011-03-21 16:39:10 UTC (rev 4321)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestAll.java 2011-03-21 16:40:13 UTC (rev 4322)
@@ -72,7 +72,8 @@
suite.addTestSuite(TestICUUnicodeKeyBuilder.class);
suite.addTestSuite(TestICUPortabilityBug.class);
-
+ suite.addTestSuite(TestICUVersionRecord.class);
+
return suite;
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestICUPortabilityBug.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestICUPortabilityBug.java 2011-03-21 16:39:10 UTC (rev 4321)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestICUPortabilityBug.java 2011-03-21 16:40:13 UTC (rev 4322)
@@ -38,11 +38,16 @@
/**
* This is a unit test for a possible ICU portability bug.
+ * <p>
+ * Note: This issue has been resolved. The problem was that someone had
+ * substituted a difference version of ICU on the classpath in the deployed
+ * system.
*
* @see https://sourceforge.net/apps/trac/bigdata/ticket/193
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
+ * @version $Id: TestICUPortabilityBug.java 4313 2011-03-18 13:16:31Z
+ * thompsonbry $
*/
public class TestICUPortabilityBug extends TestCase {
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestICUVersionRecord.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestICUVersionRecord.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestICUVersionRecord.java 2011-03-21 16:40:13 UTC (rev 4322)
@@ -0,0 +1,70 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Mar 21, 2011
+ */
+
+package com.bigdata.btree.keys;
+
+import com.bigdata.io.SerializerUtil;
+
+import junit.framework.TestCase2;
+
+/**
+ * Test suite for {@link ICUVersionRecord}
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public class TestICUVersionRecord extends TestCase2 {
+
+ /**
+ *
+ */
+ public TestICUVersionRecord() {
+ }
+
+ /**
+ * @param name
+ */
+ public TestICUVersionRecord(String name) {
+ super(name);
+ }
+
+ public void test_roundTrip() {
+
+ final ICUVersionRecord r1 = ICUVersionRecord.newInstance();
+
+ final ICUVersionRecord r2 = ICUVersionRecord.newInstance();
+
+ assertTrue(r1.equals(r2));
+
+ final ICUVersionRecord r3 = (ICUVersionRecord) SerializerUtil
+ .deserialize(SerializerUtil.serialize(r1));
+
+ assertTrue(r1.equals(r3));
+
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestICUVersionRecord.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestDiskJournal.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestDiskJournal.java 2011-03-21 16:39:10 UTC (rev 4321)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestDiskJournal.java 2011-03-21 16:40:13 UTC (rev 4322)
@@ -42,6 +42,8 @@
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
+ *
+ * @deprecated by {@link TestWORMStrategy}
*/
public class TestDiskJournal extends AbstractJournalTestCase {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
