Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[4321] trunk/bigdata/src
|
From: <tho...@us...> - 2011-03-21 16:39:17
|
Revision: 4321
http://bigdata.svn.sourceforge.net/bigdata/?rev=4321&view=rev
Author: thompsonbry
Date: 2011-03-21 16:39:10 +0000 (Mon, 21 Mar 2011)
Log Message:
-----------
Introduced an ICUVersionRecord and runtime checking of the deployed ICU version based on self-reporting via ICU's VersionInfo class. The ICUVersionRecord's address is stored in the ICommitRecord. Old stores will automatically introduce this record on their next commit. New stores will write this record when they are first created. Update to new versions of ICU will be refused based on an incompatible value for this record unless Options#UPDATE_ICUVERSION is forced to "true".
See https://sourceforge.net/apps/trac/bigdata/ticket/193
Modified Paths:
--------------
trunk/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
trunk/bigdata/src/java/com/bigdata/journal/Options.java
trunk/bigdata/src/test/com/bigdata/btree/keys/TestAll.java
trunk/bigdata/src/test/com/bigdata/btree/keys/TestICUPortabilityBug.java
Added Paths:
-----------
trunk/bigdata/src/java/com/bigdata/btree/keys/ICUVersionRecord.java
trunk/bigdata/src/test/com/bigdata/btree/keys/TestICUVersionRecord.java
Removed Paths:
-------------
trunk/bigdata/src/java/com/bigdata/journal/JournalMoveDiagnostic.java
Added: trunk/bigdata/src/java/com/bigdata/btree/keys/ICUVersionRecord.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/btree/keys/ICUVersionRecord.java (rev 0)
+++ trunk/bigdata/src/java/com/bigdata/btree/keys/ICUVersionRecord.java 2011-03-21 16:39:10 UTC (rev 4321)
@@ -0,0 +1,251 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Mar 21, 2011
+ */
+
+package com.bigdata.btree.keys;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+
+import com.bigdata.journal.Name2Addr;
+import com.ibm.icu.util.VersionInfo;
+
+/**
+ * Persistent record in which we store the version metadata for the ICU
+ * dependency in use when the journal was created. bigdata uses Unicode sort
+ * keys for various indices, including {@link Name2Addr}. A change in the ICU
+ * version can result in sort keys which are NOT compatible. Binary
+ * compatibility for Unicode sort keys is an absolute requirement for bigdata.
+ * The purpose of this persistence capable data record is to note the version of
+ * ICU against which bigdata was linked with the associated binary store file
+ * was created.
+ * <p>
+ * Note: This can result in data which apparently becomes "lost", such as this
+ * <a href="http://sourceforge.net/apps/trac/bigdata/ticket/193>trac issue</a>.
+ * The underlying problem was substituting a newer version of ICU for the one
+ * included in the bigdata distribution. Such errors are now caught by detecting
+ * a change in the ICU runtime environment.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public class ICUVersionRecord implements Externalizable {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ private VersionInfo icuVersion;
+ private VersionInfo ucolRuntimeVersion;
+ private VersionInfo ucolBuilderVersion;
+ private VersionInfo ucolTailoringsVersion;
+
+ /**
+ * The ICU software version number.
+ *
+ * @see VersionInfo#ICU_VERSION
+ */
+ public VersionInfo getICUVersion() {
+ return icuVersion;
+ }
+
+ /**
+ * If this version number changes, then the sort keys for the same Unicode
+ * string could be different.
+ *
+ * @see VersionInfo#UCOL_RUNTIME_VERSION
+ */
+ public VersionInfo getUColRuntimeVersion() {
+ return ucolRuntimeVersion;
+ }
+
+ /**
+ * If this version number changes, then the same tailoring might result in
+ * assigning different collation elements to code points (which could break
+ * binary compatibility on sort keys).
+ *
+ * @see VersionInfo#UCOL_BUILDER_VERSION
+ */
+ public VersionInfo getUColBuilderVersion() {
+ return ucolBuilderVersion;
+ }
+
+ /**
+ * The version of the collation tailorings.
+ *
+ * @see VersionInfo#UCOL_TAILORINGS_VERSION
+ */
+ public VersionInfo getUColTailoringsVersion() {
+ return ucolTailoringsVersion;
+ }
+
+ /**
+ * Factory returns a record reporting on the ICU dependency as currently
+ * linked with the code base.
+ */
+ public static ICUVersionRecord newInstance() {
+
+ final ICUVersionRecord r = new ICUVersionRecord(//
+ VersionInfo.ICU_VERSION,//
+ VersionInfo.UCOL_RUNTIME_VERSION,//
+ VersionInfo.UCOL_BUILDER_VERSION,//
+ VersionInfo.UCOL_TAILORINGS_VERSION//
+ );
+
+ return r;
+
+ }
+
+ private ICUVersionRecord(//
+ VersionInfo icuVersion,//
+ VersionInfo ucolRuntimeVesion,//
+ VersionInfo ucolBuilderVersion,//
+ VersionInfo ucolTailoringsVersion//
+ ) {
+
+ this.icuVersion = icuVersion;
+
+ this.ucolRuntimeVersion = ucolRuntimeVesion;
+
+ this.ucolBuilderVersion = ucolBuilderVersion;
+
+ this.ucolTailoringsVersion = ucolTailoringsVersion;
+
+ }
+
+ /**
+ * De-serialization contructor <strong>only</strong>.
+ */
+ public ICUVersionRecord() {
+
+ }
+
+
+ /** The initial version of this data record. */
+ private static final transient int VERSION0 = 0;
+
+ private static final transient int CURRENT_VERSION = VERSION0;
+
+ public void readExternal(ObjectInput in) throws IOException,
+ ClassNotFoundException {
+
+ final int version = in.readInt();
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new IOException("Unknown version: " + version);
+ }
+
+ icuVersion = readVersionInfo(in);
+ ucolRuntimeVersion = readVersionInfo(in);
+ ucolBuilderVersion = readVersionInfo(in);
+ ucolTailoringsVersion = readVersionInfo(in);
+
+ }
+
+ public void writeExternal(ObjectOutput out) throws IOException {
+
+ out.writeInt(CURRENT_VERSION);
+ writeVersionInfo(icuVersion, out);
+ writeVersionInfo(ucolRuntimeVersion, out);
+ writeVersionInfo(ucolBuilderVersion, out);
+ writeVersionInfo(ucolTailoringsVersion, out);
+
+ }
+
+ private VersionInfo readVersionInfo(final ObjectInput in) throws IOException {
+
+ final int major = in.readInt();
+ final int minor = in.readInt();
+ final int milli = in.readInt();
+ final int micro = in.readInt();
+
+ return VersionInfo.getInstance(major, minor, milli, micro);
+
+ }
+
+ private void writeVersionInfo(final VersionInfo v, final ObjectOutput out)
+ throws IOException {
+
+ out.writeInt(v.getMajor());
+ out.writeInt(v.getMinor());
+ out.writeInt(v.getMicro());
+ out.writeInt(v.getMilli());
+
+ }
+
+ /**
+ * A human readable representation of the data record.
+ */
+ public String toString() {
+ final StringBuilder sb = new StringBuilder();
+ sb.append(getClass().getName());
+ sb.append("{icuVersion=" + icuVersion);
+ sb.append(",ucolRuntimeVersion=" + ucolRuntimeVersion);
+ sb.append(",ucolBuilderVersion=" + ucolBuilderVersion);
+ sb.append(",ucolTailoringsVersion=" + ucolTailoringsVersion);
+ sb.append("}");
+ return sb.toString();
+ }
+
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ public boolean equals(final Object o) {
+ if (this == o)
+ return true;
+ if (!(o instanceof ICUVersionRecord))
+ return false;
+ final ICUVersionRecord r = (ICUVersionRecord) o;
+ if (!icuVersion.equals(r.icuVersion))
+ return false;
+ if (!ucolRuntimeVersion.equals(r.ucolRuntimeVersion))
+ return false;
+ if (!ucolBuilderVersion.equals(r.ucolBuilderVersion))
+ return false;
+ if (!ucolTailoringsVersion.equals(r.ucolTailoringsVersion))
+ return false;
+ return true;
+ }
+
+ /**
+ * Writes out the {@link ICUVersionRecord} for the current classpath.
+ *
+ * @param args
+ * Ignored.
+ */
+ static public void main(String[] args) {
+
+ System.out.println(ICUVersionRecord.newInstance().toString());
+
+ }
+
+}
Property changes on: trunk/bigdata/src/java/com/bigdata/btree/keys/ICUVersionRecord.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: trunk/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2011-03-21 16:36:49 UTC (rev 4320)
+++ trunk/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2011-03-21 16:39:10 UTC (rev 4321)
@@ -51,6 +51,7 @@
import com.bigdata.btree.IndexMetadata;
import com.bigdata.btree.IndexSegment;
import com.bigdata.btree.ReadOnlyIndex;
+import com.bigdata.btree.keys.ICUVersionRecord;
import com.bigdata.cache.ConcurrentWeakValueCache;
import com.bigdata.cache.ConcurrentWeakValueCacheWithTimeout;
import com.bigdata.cache.HardReferenceQueue;
@@ -62,6 +63,7 @@
import com.bigdata.config.LongValidator;
import com.bigdata.counters.CounterSet;
import com.bigdata.counters.Instrument;
+import com.bigdata.io.SerializerUtil;
import com.bigdata.journal.Name2Addr.Entry;
import com.bigdata.mdi.IResourceMetadata;
import com.bigdata.mdi.JournalMetadata;
@@ -167,6 +169,13 @@
public static transient final int ROOT_NAME2ADDR = 0;
/**
+ * The index of the root address containing the {@link ICUVersionRecord}.
+ * That record specifies the ICU version metadata which was in force when
+ * the journal was created.
+ */
+ public static transient final int ROOT_ICUVERSION = 3;
+
+ /**
* A clone of the properties used to initialize the {@link Journal}.
*/
final protected Properties properties;
@@ -273,6 +282,11 @@
private volatile CommitRecordIndex _commitRecordIndex;
/**
+ * The {@link ICUVersionRecord} iff known.
+ */
+ private volatile ICUVersionRecord _icuVersionRecord;
+
+ /**
* The configured capacity for the {@link HardReferenceQueue} backing the
* index cache maintained by the "live" {@link Name2Addr} object.
*
@@ -1148,6 +1162,27 @@
// new or re-load commit record index from store via root block.
this._commitRecordIndex = _getCommitRecordIndex();
+ // new or re-load from the store.
+ this._icuVersionRecord = _getICUVersionRecord();
+
+ // verify the ICU version.
+ if (this._icuVersionRecord != null
+ && !ICUVersionRecord.newInstance().equals(
+ this._icuVersionRecord)) {
+
+ final boolean update = Boolean.valueOf(properties.getProperty(
+ Options.UPDATE_ICU_VERSION, "false"));
+
+ if (!update) {
+
+ throw new RuntimeException("ICUVersionChange: store="
+ + this._icuVersionRecord + ", runtime="
+ + ICUVersionRecord.newInstance());
+
+ }
+
+ }
+
// Give the store a chance to set any committers that it defines.
setupCommitters();
@@ -2061,6 +2096,9 @@
// clear reference and reload from the store.
_commitRecordIndex = _getCommitRecordIndex();
+ // clear reference and reload from the store.
+ _icuVersionRecord = _getICUVersionRecord();
+
// clear the array of committers.
_committers = new ICommitter[_committers.length];
@@ -2609,8 +2647,112 @@
setupName2AddrBTree(getRootAddr(ROOT_NAME2ADDR));
+ /*
+ * Responsible for writing the ICUVersionRecord exactly once onto
+ * the backing store, e.g., when the store is created or when it is
+ * open with the "update" option specified for ICU.
+ */
+ setCommitter(ROOT_ICUVERSION, new ICUVersionCommitter());
+
}
+ /**
+ * Return the {@link ICUVersionRecord} from the current
+ * {@link ICommitRecord} -or- a new instance for the current runtime
+ * environment if the root address for {@link #ROOT_ICUVERSION} is
+ * {@link #NULL}.
+ */
+ private ICUVersionRecord _getICUVersionRecord() {
+
+ assert _fieldReadWriteLock.writeLock().isHeldByCurrentThread();
+
+ final long addr = getRootAddr(ROOT_ICUVERSION);
+
+ final ICUVersionRecord r;
+ if (addr == NULL) {
+ // New instance for the current runtime environment.
+ r = ICUVersionRecord.newInstance();
+ } else {
+ // Existing instance from the store.
+ r = (ICUVersionRecord) SerializerUtil.deserialize(read(addr));
+ }
+ return r;
+
+ }
+
+ /**
+ * Writes the {@link ICUVersionRecord} onto the store iff either (a) it does
+ * not exist; or (b) it exists, it differs from the last persistent record,
+ * and the update flag was specified.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan
+ * Thompson</a>
+ *
+ * @see Options#UPDATE_ICU_VERSION
+ */
+ private class ICUVersionCommitter implements ICommitter {
+
+ private boolean update;
+
+ private long lastAddr;
+
+ private ICUVersionCommitter() {
+
+ // the "update" option.
+ update = Boolean.valueOf(properties.getProperty(
+ Options.UPDATE_ICU_VERSION, "false"));
+
+ // lookup the address of the ICU version record (may be NULL).
+ lastAddr = getRootAddr(ROOT_ICUVERSION);
+
+ }
+
+ /**
+ * Commits a new {@link ICUVersionRecord} IF none is defined -OR- IF one
+ * is defined, it is a different version of ICU, and the update flag is
+ * set.
+ */
+ public long handleCommit(final long commitTime) {
+
+ if(!update && lastAddr != NULL) {
+
+ // Nothing changed.
+ return lastAddr;
+
+ }
+
+ /*
+ * Note: The Journal only validates the persistent ICU version
+ * record in its constructor. By the time the code reaches this
+ * point, it is either in agreement or will be written.
+ */
+
+ final ICUVersionRecord r = ICUVersionRecord.newInstance();
+
+ if (lastAddr == NULL || !(r.equals(_icuVersionRecord) && update)) {
+
+ if (_icuVersionRecord != null && update)
+ log.warn("Updating ICUVersion: old=" + _icuVersionRecord
+ + ", new=" + r);
+
+ // do not update next time.
+ update = false;
+
+ // write ICU version record onto the store.
+ lastAddr = write(ByteBuffer.wrap(SerializerUtil.serialize(r)));
+
+ // return address of the ICU version record.
+ return lastAddr;
+
+ }
+
+ // Nothing changed.
+ return lastAddr;
+
+ }
+
+ }
+
/*
* named indices.
*/
Deleted: trunk/bigdata/src/java/com/bigdata/journal/JournalMoveDiagnostic.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/journal/JournalMoveDiagnostic.java 2011-03-21 16:36:49 UTC (rev 4320)
+++ trunk/bigdata/src/java/com/bigdata/journal/JournalMoveDiagnostic.java 2011-03-21 16:39:10 UTC (rev 4321)
@@ -1,537 +0,0 @@
-/**
-
-Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-/*
- * Created on Mar 13, 2011
- */
-
-package com.bigdata.journal;
-
-import java.io.File;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.log4j.Logger;
-import org.openrdf.model.impl.URIImpl;
-import org.openrdf.model.vocabulary.RDF;
-import org.openrdf.model.vocabulary.RDFS;
-
-import com.bigdata.btree.BytesUtil;
-import com.bigdata.btree.IIndex;
-import com.bigdata.btree.ITuple;
-import com.bigdata.btree.ITupleIterator;
-import com.bigdata.btree.keys.CollatorEnum;
-import com.bigdata.btree.keys.DecompositionEnum;
-import com.bigdata.btree.keys.DefaultKeyBuilderFactory;
-import com.bigdata.btree.keys.IKeyBuilder;
-import com.bigdata.btree.keys.IKeyBuilderFactory;
-import com.bigdata.btree.keys.KeyBuilder;
-import com.bigdata.btree.keys.StrengthEnum;
-import com.bigdata.rawstore.Bytes;
-import com.bigdata.rdf.axioms.NoAxioms;
-import com.bigdata.rdf.sail.BigdataSail;
-import com.bigdata.rdf.sail.BigdataSail.BigdataSailConnection;
-import com.bigdata.rdf.vocab.NoVocabulary;
-
-/**
- * A diagnostic utility for problems with Unicode collation issues which can
- * appear when a journal is moved to another machine. This utility is designed
- * to be run on both the source machine and the target machine. It reports back
- * specific key values from the {@link Name2Addr} index, metadata about the
- * Unicode collation rules in use for that index, and metadata about the
- * {@link Locale} as self-reported by the JVM. These information are intended
- * for analysis in support of a trouble ticket.
- * <p>
- * Note: This issue was resolved. The underlying problem was that someone had
- * substituted a different ICU dependency (v4.4.1, which is not even a stable
- * release as opposed to v3.6, which is what we distributed).
- *
- * @see https://sourceforge.net/apps/trac/bigdata/ticket/193
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id: JournalMoveDiagnostic.java 4294 2011-03-14 15:29:50Z
- * thompsonbry $
- */
-public class JournalMoveDiagnostic {
-
- private static final Logger log = Logger
- .getLogger(JournalMoveDiagnostic.class);
-
- /**
- * You must specify the name of the Journal file. In addition, you may
- * specify one or more index names. If no index names are specified, it will
- * report metadata for all Name2Addr entries.
- *
- * @param args
- * <code>
- * journalFile (indexName)*
- * </code>
- *
- * @throws Exception
- */
- public static void main(final String[] args) throws Exception {
-
- if (args.length == 0) {
-
- System.err.println("usage: <filename> (indexName)*");
-
- System.exit(1);
-
- }
-
- final File journalFile = new File(args[0]);
-
- if (Boolean.valueOf(System.getProperty(
- "com.bigdata.journal.JournalMoveDiagnostic.deleteFirst",
- "false"))) {
-
- if (journalFile.exists()) {
-
- System.out.println("\n\nDELETING OLD JOURNAL: " + journalFile);
-
- if (!journalFile.delete()) {
-
- System.err.println("Could not delete old journal: "
- + journalFile);
-
- }
-
- }
-
- }
-
- if (!journalFile.exists()) {
-
- System.out.println("\n\nCREATING NEW JOURNAL "+journalFile);
-
- final Properties properties = new Properties();
-
- // use the named file.
- properties.setProperty(Options.FILE, journalFile.toString());
-
- // use a small initial journal size to keep down the artifact size.
- properties.setProperty(Options.INITIAL_EXTENT,""+(Bytes.megabyte32*1));
-
- // triples only w/o inference.
- properties.setProperty(BigdataSail.Options.AXIOMS_CLASS, NoAxioms.class.getName());
- properties.setProperty(BigdataSail.Options.VOCABULARY_CLASS, NoVocabulary.class.getName());
- properties.setProperty(BigdataSail.Options.TRUTH_MAINTENANCE, "false");
- properties.setProperty(BigdataSail.Options.JUSTIFY, "false");
- properties.setProperty(BigdataSail.Options.TEXT_INDEX, "false");
-
- final BigdataSail sail = new BigdataSail(properties);
-
- try {
-
- sail.initialize();
-
- final BigdataSailConnection cxn = sail.getConnection();
-
- try {
-
- // add a single statement.
- cxn.addStatement(//
- new URIImpl("http://www.bigdata.com"),
- RDF.TYPE, //
- RDFS.RESOURCE
- );
-
- cxn.commit();
-
- } finally {
-
- cxn.close();
-
- }
-
- } finally {
-
- sail.shutDown();
-
- }
-
-// System.err.println("Not found: " + journalFile);
-//
-// System.exit(1);
-
- }
-
- {
-
- System.err.println("Default Locale: " + dumpLocale(Locale.getDefault()));
-
-// for (Locale tmp : Locale.getAvailableLocales())
-// System.err.println("Available Locale: " + tmp);\
-
- }
-
- // collect the set of index names on which we will report.
- final Set<String> indexNames = new LinkedHashSet<String>();
-
- for (int i = 1; i < args.length; i++) {
-
- indexNames.add(args[i]);
-
- }
-
- final Properties properties = new Properties();
- {
-
- properties.setProperty(Options.FILE, journalFile.toString());
-
- properties.setProperty(Options.READ_ONLY, "" + true);
-
- // FIXME We should auto-discover this from the root blocks!
- properties.setProperty(Options.BUFFER_MODE,BufferMode.Disk.toString());
-
- }
-
- System.err.println("Opening (read-only): " + journalFile);
-
- final Journal jnl = new Journal(properties);
-
- try {
-
- dumpName2Addr(jnl, indexNames, jnl.getLastCommitTime());
-
- } finally {
-
- jnl.shutdownNow();
-
- }
-
- }
-
- /**
- * Dump out all data associated with the {@link Locale}.
- *
- * @param l
- * The {@link Locale}.
- *
- * @return A string representation of its data.
- */
- private static final String dumpLocale(final Locale l) {
-
- final StringBuilder sb = new StringBuilder();
-
- sb.append("\n Locale : [" + l + "]");
- sb.append("\n Country : [" + l.getCountry() + "]");
- sb.append("\n Language : [" + l.getLanguage() + "]");
- sb.append("\n Variant : [" + l.getVariant() + "]");
- sb.append("\n ISO3 Country : [" + l.getISO3Country() + "]");
- sb.append("\n ISO3 Language: [" + l.getISO3Language() + "]");
- sb.append("\n");
-
- return sb.toString();
-
- }
-
- /**
- * Dump out some detailed information about the {@link Name2Addr} index, the
- * manner in which it should be encoding Unicode Strings into unsigned
- * byte[] keys, and, for each named index, the actual index name, the actual
- * unsigned byte[] key found in the Name2Addr index, and the unsigned byte[]
- * key under which the machine on which this utility is running would
- * attempt to resolve the index name - this last key SHOULD be the same as
- * the key under which the index entry was found. If it is NOT the same then
- * this indicates an error in the way in which the keys are being generated
- * from the index names. Information is written onto stderr.
- *
- * @param jnl
- * The journal.
- * @param indexNames
- * The name of one or more indices on which the per-index
- * metadata will be reported.
- * @param timestamp
- * The timestamp of the commit record for which this information
- * will be reported.
- */
- private static final void dumpName2Addr(final Journal jnl,
- final Set<String> indexNames, final long timestamp) {
-
- final IIndex name2Addr = jnl.getName2Addr(timestamp);
-
- // The key builder actually used by Name2Addr.
- final IKeyBuilder theKeyBuilder;
- // A key builder from the default factory.
- final IKeyBuilder aKeyBuilder;
- {
- /*
- * Show the key builder factory that the Name2Addr instance is
- * actually using (this shows the tupleSerializer, but that shows
- * the key builder factory which is what we really care about).
- */
- theKeyBuilder = name2Addr.getIndexMetadata().getKeyBuilder();
-
- /*
- * A key builder factory as it would be configured on this machine
- * for a new Name2Addr index, e.g., if we created a new Journal.
- */
- final IKeyBuilderFactory aKeyBuilderFactory = new DefaultKeyBuilderFactory(
- new Properties());
-
- System.err.println("KeyBuilderFactory if created new:\n"
- + aKeyBuilderFactory);
-
- /*
- * A key builder generated by that factory. This key builder should
- * have the same behavior that we observe for Name2Addr IF the
- * KeyBuilderFactory inherits the same Locale, [collator],
- * [strength], and [decompositionMode] attributes which were used to
- * create the Journal. Differences in Locale (e.g., language),
- * collator (e.g., JDK versus ICU), strength (e.g., IDENTICAL vs
- * PRIMARY), or decompositionMode (e.g., None versus Full) can all
- * cause the unsigned byte[] keys generated by this key builder to
- * differ from those generated on the machine where (and when) the
- * journal was originally created.
- */
- aKeyBuilder = aKeyBuilderFactory.getKeyBuilder();
-
- System.err.println("Name2Addr effective key builder:\n"
- + theKeyBuilder);
-
- System.err.println("Name2Addr if-new key builder:\n"
- + aKeyBuilder);
- }
-
- // Names of indices and the #of times they were found.
- final Map<String, AtomicInteger> dups = new LinkedHashMap<String, AtomicInteger>();
-
- // the named indices
- final ITupleIterator<?> itr = name2Addr.rangeIterator();
-
- while (itr.hasNext()) {
-
- final ITuple<?> tuple = itr.next();
-
- /*
- * A registered index. Entry.name is the actual name for the index
- * and is serialized using Java default serialization as a String.
- * The key for the entry in the Name2Addr index should be the
- * Unicode sort key for Entry.name. That Unicode sort key should be
- * generated by the collation rules as defined by the IndexMetadata
- * record for the Name2Addr index.
- */
- final Name2Addr.Entry entry = Name2Addr.EntrySerializer.INSTANCE
- .deserialize(tuple.getValueStream());
-
- // Track #of times we visit an index having this name.
- {
-
- AtomicInteger tmp = dups.get(entry.name);
-
- if (tmp == null) {
-
- dups.put(entry.name, tmp = new AtomicInteger(0));
-
- }
-
- tmp.incrementAndGet();
-
- }
-
- if (!indexNames.isEmpty() && !indexNames.contains(entry.name)) {
- /*
- * A specific set of index names was given and this is not one
- * of those indices.
- */
- continue;
- }
-
- System.err.println("-----");
-
- System.err.println("Considering: " + tuple);
-
- /*
- * The actual unsigned byte[] under which the Name2Addr entry is
- * indexed.
- */
- final byte[] theKey = tuple.getKey();
-
- /*
- * Using the TupleSerializer for the Name2Addr index, generate the
- * Unicode sort key for Entry.name. This *should* be the same as the
- * unsigned byte[] key for the tuple in the Name2Addr index. If it
- * is NOT the same, then there is a problem with the preservation of
- * the Unicode collation rules such that the same input string
- * (Entry.name) is resulting in a different unsigned byte[] key. If
- * this happens, then the indices can appear to become "lost"
- * because the "spelling rules" for the Name2Addr index have
- * changed.
- *
- * @see https://sourceforge.net/apps/trac/bigdata/ticket/193
- */
- final byte[] b = name2Addr.getIndexMetadata().getTupleSerializer()
- .serializeKey(entry.name);
- final byte[] b2 = theKeyBuilder.reset().append(entry.name).getKey();
- if(!BytesUtil.bytesEqual(b, b2)) {
- System.err.println("ERROR: tupleSer and keyBuilder do not agree");
- }
-
-// /*
-// * This uses the key builder which would be created for a new
-// * Name2Addr instance on this host.
-// */
-// final byte[] c = aKeyBuilder.reset().append(entry.name).getKey();
-
- System.err.println("name=" + entry.name);
-
- System.err.println("tuple : " + BytesUtil.toString(theKey));
-
- final boolean consistent = BytesUtil.bytesEqual(theKey, b);
-
-// final boolean consistent2 = BytesUtil.bytesEqual(theKey,c);
-
- if (!consistent) {
- /*
- * The Name2Addr index has an entry which we will be unable to
- * locate when given the name of the index because the generated
- * unsigned byte[] key is NOT the same as the unsigned byte[]
- * key under which the Entry is stored in the index.
- */
- System.err.println("recode: " + BytesUtil.toString(b));
- System.err.println("ERROR : Name2Addr inconsistent for ["
- + entry.name + "]");
- searchForConsistentConfiguration(entry.name, theKey);
- }
-// if (!consistent2) {
-// /*
-// * @todo javadoc.
-// */
-// System.err.println("recod2: " + BytesUtil.toString(c));
-// System.err.println("ERROR : Name2Addr inconsistent for ["
-// + entry.name + "]");
-// }
-
- }
-
- System.err.println("\n===========");
-
- /*
- * Show any indices for which are have more than one entry. There is
- * an encoding problem for the names of any such indices.
- */
- for (Map.Entry<String, AtomicInteger> e : dups.entrySet()) {
-
- if (e.getValue().get() != 1) {
-
- System.err.println("ERROR: name=[" + e.getKey() + "] has "
- + e.getValue().get() + " Name2Addr entries.");
-
- }
-
- }
-
- } // dumpName2Addr
-
- /**
- * Search for a configuration of an {@link IKeyBuilderFactory} which is
- * consistent with the given key when encoding the given string into an
- * unsigned byte[].
- *
- * @param str
- * The given string.
- * @param expected
- * The given key.
- */
- private static void searchForConsistentConfiguration(final String str,
- final byte[] expected) {
-
-// final byte[] expected = keyBuilder.reset().append(str).getKey();
-
- // To test all.
- final Locale[] locales = Locale.getAvailableLocales();
- // To test just the default locale.
-// final Locale[] locales = new Locale[]{Locale.getDefault()};
-
- int nconsistent = 0;
-
- // Consider each Locale
- for(Locale l : locales) {
-
- // Consider all Collator implementations (JDK, ICU, ICU4JNI)
- for(CollatorEnum c : CollatorEnum.values()) {
-
- // Consider all Sollator strengths.
- for(StrengthEnum s : StrengthEnum.values()) {
-
- // Consider all Collator decomposition modes.
- for(DecompositionEnum d : DecompositionEnum.values()) {
-
- // Setup the collator.
- final Properties p = new Properties();
- p.setProperty(KeyBuilder.Options.USER_COUNTRY, l.getCountry());
- p.setProperty(KeyBuilder.Options.USER_LANGUAGE, l.getLanguage());
- p.setProperty(KeyBuilder.Options.USER_VARIANT, l.getVariant());
- p.setProperty(KeyBuilder.Options.COLLATOR, c.toString());
- p.setProperty(KeyBuilder.Options.STRENGTH, s.toString());
- p.setProperty(KeyBuilder.Options.DECOMPOSITION, d.toString());
-
- final IKeyBuilderFactory f;
- final IKeyBuilder tmp;
- try {
- f = new DefaultKeyBuilderFactory(p);
- tmp = f.getKeyBuilder();
- } catch (IllegalArgumentException t) {
- if (log.isDebugEnabled())
- log.debug("Illegal configuration: " + t);
- continue;
- } catch (UnsupportedOperationException t) {
- if (log.isDebugEnabled())
- log.debug("Illegal configuration: " + t);
- continue;
- }
-
- final byte[] actual = tmp.reset().append(str).getKey();
-
- if (BytesUtil.bytesEqual(expected, actual)) {
-
- System.out
- .println("Consistent configuration: " + p);
-
- nconsistent++;
-
- }
-
- }
-
- }
-
- }
-
- }
-
- if (nconsistent == 0) {
-
- System.err.println("No consistent configuration was found.");
-
- }
-
- } // searchForConsistentConfiguration()
-
-}
Modified: trunk/bigdata/src/java/com/bigdata/journal/Options.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/journal/Options.java 2011-03-21 16:36:49 UTC (rev 4320)
+++ trunk/bigdata/src/java/com/bigdata/journal/Options.java 2011-03-21 16:39:10 UTC (rev 4321)
@@ -32,6 +32,7 @@
import com.bigdata.btree.Checkpoint;
import com.bigdata.btree.IndexSegment;
+import com.bigdata.btree.keys.ICUVersionRecord;
import com.bigdata.cache.HardReferenceQueue;
import com.bigdata.io.DirectBufferPool;
import com.bigdata.io.FileLockUtility;
@@ -324,6 +325,19 @@
String ALTERNATE_ROOT_BLOCK = AbstractJournal.class.getName()+".alternateRootBlock";
/**
+ * <strong>WARNING - The use of this option is dangerous.</strong> This
+ * option may be used to update the {@link ICUVersionRecord} associated with
+ * the journal. ICU provides a Unicode sort key generation service for
+ * bigdata. Unicode sort keys are used in many indices, including the
+ * {@link Name2Addr} index. If the new ICU version produces Unicode sort
+ * keys which are not binary compatible with the Journal, then your data may
+ * become inaccessible since you will be unable to probe the
+ * {@link Name2Addr} index to locate named indices. The same problem can
+ * manifest with application indices which use Unicode sort keys.
+ */
+ String UPDATE_ICU_VERSION = AbstractJournal.class.getName()+".updateICUVersion";
+
+ /**
* An optional boolean property (default is {@value #DEFAULT_CREATE}). When
* <code>true</code> and the named file is not found, a new journal will be
* created. If the file exists but is empty, then a new journal will be
Modified: trunk/bigdata/src/test/com/bigdata/btree/keys/TestAll.java
===================================================================
--- trunk/bigdata/src/test/com/bigdata/btree/keys/TestAll.java 2011-03-21 16:36:49 UTC (rev 4320)
+++ trunk/bigdata/src/test/com/bigdata/btree/keys/TestAll.java 2011-03-21 16:39:10 UTC (rev 4321)
@@ -72,6 +72,7 @@
suite.addTestSuite(TestICUUnicodeKeyBuilder.class);
suite.addTestSuite(TestICUPortabilityBug.class);
+ suite.addTestSuite(TestICUVersionRecord.class);
return suite;
Modified: trunk/bigdata/src/test/com/bigdata/btree/keys/TestICUPortabilityBug.java
===================================================================
--- trunk/bigdata/src/test/com/bigdata/btree/keys/TestICUPortabilityBug.java 2011-03-21 16:36:49 UTC (rev 4320)
+++ trunk/bigdata/src/test/com/bigdata/btree/keys/TestICUPortabilityBug.java 2011-03-21 16:39:10 UTC (rev 4321)
@@ -38,11 +38,16 @@
/**
* This is a unit test for a possible ICU portability bug.
+ * <p>
+ * Note: This issue has been resolved. The problem was that someone had
+ * substituted a difference version of ICU on the classpath in the deployed
+ * system.
*
* @see https://sourceforge.net/apps/trac/bigdata/ticket/193
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
+ * @version $Id: TestICUPortabilityBug.java 4314 2011-03-18 14:31:57Z
+ * thompsonbry $
*/
public class TestICUPortabilityBug extends TestCase {
Added: trunk/bigdata/src/test/com/bigdata/btree/keys/TestICUVersionRecord.java
===================================================================
--- trunk/bigdata/src/test/com/bigdata/btree/keys/TestICUVersionRecord.java (rev 0)
+++ trunk/bigdata/src/test/com/bigdata/btree/keys/TestICUVersionRecord.java 2011-03-21 16:39:10 UTC (rev 4321)
@@ -0,0 +1,70 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Mar 21, 2011
+ */
+
+package com.bigdata.btree.keys;
+
+import com.bigdata.io.SerializerUtil;
+
+import junit.framework.TestCase2;
+
+/**
+ * Test suite for {@link ICUVersionRecord}
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public class TestICUVersionRecord extends TestCase2 {
+
+ /**
+ *
+ */
+ public TestICUVersionRecord() {
+ }
+
+ /**
+ * @param name
+ */
+ public TestICUVersionRecord(String name) {
+ super(name);
+ }
+
+ public void test_roundTrip() {
+
+ final ICUVersionRecord r1 = ICUVersionRecord.newInstance();
+
+ final ICUVersionRecord r2 = ICUVersionRecord.newInstance();
+
+ assertTrue(r1.equals(r2));
+
+ final ICUVersionRecord r3 = (ICUVersionRecord) SerializerUtil
+ .deserialize(SerializerUtil.serialize(r1));
+
+ assertTrue(r1.equals(r3));
+
+ }
+
+}
Property changes on: trunk/bigdata/src/test/com/bigdata/btree/keys/TestICUVersionRecord.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
