From: Wolfgang M. M. <wol...@us...> - 2004-05-10 11:23:24
|
Update of /cvsroot/exist/eXist-1.0/src/org/exist/storage In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv25553/src/org/exist/storage Modified Files: DBBroker.java IndexPaths.java NativeElementIndex.java NativeBroker.java Added Files: NodePath.java Log Message: XUpdate: * Btree not correctly updated after node changes, leading to NullPointerException * Attribute values truncated * Page splitting error results in NullPointerException after a few hundred XUpdates * New configuration parameter to control XUpdate behaviour Other: * Properties passed to Collection, XPathQueryService, XQueryService had no effect. * Modified evaluation of index settings in conf.xml. --- NEW FILE: NodePath.java --- /* * eXist Open Source Native XML Database * Copyright (C) 2001-04 Wolfgang M. Meier * wol...@ex... * http://exist-db.org * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. * * $Id: NodePath.java,v 1.1 2004/05/10 11:22:41 wolfgang_m Exp $ */ package org.exist.storage; import java.util.StringTokenizer; import org.exist.util.FastStringBuffer; import org.exist.util.MutableStringTokenizer; /** * @author wolf */ public class NodePath { public final static String WILDCARD = "*"; private String[] components = new String[5]; private int pos = 0; public NodePath() { } /** * */ public NodePath(String path) { init(path); } public void addComponent(String component) { if(pos == components.length) { String[] t = new String[pos + 1]; System.arraycopy(components, 0, t, 0, pos); components = t; } components[pos++] = component; } public void removeLastComponent() { components[--pos] = null; } public final boolean match(NodePath other) { boolean skip = false; int i = 0; for(int j = 0; j < other.pos; j++) { if(i == pos) return true; if(components[i].equals(WILDCARD)) { ++i; skip = true; } if(other.components[j].equals(components[i])) { ++i; skip = false; } else if(skip) { continue; } else return false; } return i == pos; } public final boolean match( CharSequence other) { int i = 0; boolean skip = false; MutableStringTokenizer tokenizer = new MutableStringTokenizer(other, "/"); CharSequence next; while((next = tokenizer.nextToken()) != null) { if(i == pos) return true; if(components[i].equals(WILDCARD)) { ++i; skip = true; } if(next.equals(components[i])) { ++i; skip = false; } else if(skip) { continue; } else return false; } return i == pos; } public void reset() { for(int i = 0; i < components.length; i++) components[i] = null; pos = 0; } private void init( String path ) { StringTokenizer tokenizer = new StringTokenizer(path, "/"); FastStringBuffer token = new FastStringBuffer(); String next; int pos = 0; char ch = 0; final int pathLen = path.length(); while ( pos < pathLen ) { ch = path.charAt(pos); switch ( ch ) { case '/': next = token.toString(); token.reset(); if ( next.length( ) > 0 ) addComponent( next ); if ( path.charAt( ++pos ) == '/' ) addComponent( WILDCARD ); break; default: token.append(ch); pos++; } } if ( token.length() > 0 ) addComponent( token.toString() ); } } Index: NativeBroker.java =================================================================== RCS file: /cvsroot/exist/eXist-1.0/src/org/exist/storage/NativeBroker.java,v retrieving revision 1.65 retrieving revision 1.66 diff -C2 -d -r1.65 -r1.66 *** NativeBroker.java 3 May 2004 13:08:43 -0000 1.65 --- NativeBroker.java 10 May 2004 11:22:41 -0000 1.66 *************** *** 110,113 **** --- 110,114 ---- private static final String ROOT_COLLECTION = "/db"; private static final String EXCEPTION_DURING_REINDEX = "exception during reindex"; + /** default buffer size setting */ protected final static int BUFFERS = 256; *************** *** 116,133 **** protected final static int MEM_LIMIT_CHECK = 10000; protected CollectionStore collectionsDb = null; protected DOMFile domDb = null; protected NativeElementIndex elementIndex; - protected ElementPool elementPool = new ElementPool(50); protected BFile elementsDb = null; protected NativeTextEngine textEngine; protected Serializer xmlSerializer; protected PatternCompiler compiler = new Perl5Compiler(); protected PatternMatcher matcher = new Perl5Matcher(); protected int defaultIndexDepth = 1; protected Map idxPathMap; protected boolean readOnly = false; protected int memMinFree; protected int nodesCount = 0; private final Runtime run = Runtime.getRuntime(); --- 117,142 ---- protected final static int MEM_LIMIT_CHECK = 10000; + // the database files protected CollectionStore collectionsDb = null; protected DOMFile domDb = null; protected NativeElementIndex elementIndex; protected BFile elementsDb = null; + protected NativeTextEngine textEngine; protected Serializer xmlSerializer; + protected PatternCompiler compiler = new Perl5Compiler(); protected PatternMatcher matcher = new Perl5Matcher(); + protected int defaultIndexDepth = 1; protected Map idxPathMap; + protected boolean readOnly = false; + protected int memMinFree; + + // used to count the nodes inserted after the last memory check protected int nodesCount = 0; + private final Runtime run = Runtime.getRuntime(); *************** *** 154,157 **** --- 163,167 ---- if ((memMinFree = config.getInteger("db-connection.min_free_memory")) < 0) memMinFree = 5000000; + Paged.setPageSize(pageSize); String pathSep = System.getProperty("file.separator", "/"); *************** *** 242,245 **** --- 252,256 ---- if (readOnly) LOG.info("database runs in read-only mode"); + idxPathMap = (Map) config.getProperty("indexer.map"); textEngine = new NativeTextEngine(this, config, buffers); *************** *** 383,387 **** public NodeSet findElementsByTagName(byte type, DocumentSet docs, QName qname, NodeSelector selector) { ! // final long start = System.currentTimeMillis(); final ExtArrayNodeSet result = new ExtArrayNodeSet(docs.getLength(), 256); DocumentImpl doc; --- 394,398 ---- public NodeSet findElementsByTagName(byte type, DocumentSet docs, QName qname, NodeSelector selector) { ! final long start = System.currentTimeMillis(); final ExtArrayNodeSet result = new ExtArrayNodeSet(docs.getLength(), 256); DocumentImpl doc; *************** *** 614,618 **** domDb.setOwnerObject(this); try { ! return new NodeIterator(this, domDb, proxy, true); } catch (BTreeException e) { LOG.debug("failed to create node iterator", e); --- 625,629 ---- domDb.setOwnerObject(this); try { ! return new NodeIterator(this, domDb, proxy, false); } catch (BTreeException e) { LOG.debug("failed to create node iterator", e); *************** *** 988,992 **** for (Iterator i = nodes.iterator(); i.hasNext();) { ref = (Value) i.next(); ! gid = ByteConversion.byteToLong(ref.data(), 4); if (oldDoc.getTreeLevel(gid) >= doc.reindexRequired()) { if (node != null) { --- 999,1003 ---- for (Iterator i = nodes.iterator(); i.hasNext();) { ref = (Value) i.next(); ! gid = ByteConversion.byteToLong(ref.data(), ref.start() + 4); if (oldDoc.getTreeLevel(gid) >= doc.reindexRequired()) { if (node != null) { *************** *** 1020,1024 **** new NodeProxy(doc, n.getGID(), n.getInternalAddress())); iterator.next(); ! scanNodes(iterator, n, new StringBuffer()); } } else { --- 1031,1035 ---- new NodeProxy(doc, n.getGID(), n.getInternalAddress())); iterator.next(); ! scanNodes(iterator, n, new NodePath()); } } else { *************** *** 1042,1046 **** * @param currentPath */ ! private void reindex(final NodeImpl node, StringBuffer currentPath) { if (node.getGID() < 0) LOG.debug("illegal node: " + node.getGID() + "; " + node.getNodeName()); --- 1053,1057 ---- * @param currentPath */ ! private void reindex(final NodeImpl node, NodePath currentPath) { if (node.getGID() < 0) LOG.debug("illegal node: " + node.getGID() + "; " + node.getNodeName()); *************** *** 1049,1055 **** final short nodeType = node.getNodeType(); final long gid = node.getGID(); - final String nodeName = node.getNodeName(); final DocumentImpl doc = (DocumentImpl) node.getOwnerDocument(); - final byte data[] = node.serialize(); final int depth = idx == null ? defaultIndexDepth : idx.getIndexDepth(); final int level = doc.getTreeLevel(gid); --- 1060,1064 ---- *************** *** 1085,1089 **** qname.setNameType(ElementValue.ELEMENT); tempProxy.setHasIndex( ! idx == null || idx.match(currentPath.toString())); elementIndex.setDocument(doc); elementIndex.addRow(qname, tempProxy); --- 1094,1098 ---- qname.setNameType(ElementValue.ELEMENT); tempProxy.setHasIndex( ! idx == null || idx.match(currentPath)); elementIndex.setDocument(doc); elementIndex.addRow(qname, tempProxy); *************** *** 1091,1095 **** case Node.ATTRIBUTE_NODE : tempProxy.setHasIndex( ! idx == null || idx.match(currentPath.toString())); elementIndex.setDocument(doc); qname = --- 1100,1104 ---- case Node.ATTRIBUTE_NODE : tempProxy.setHasIndex( ! idx == null || idx.match(currentPath)); elementIndex.setDocument(doc); qname = *************** *** 1102,1108 **** // check if attribute value should be fulltext-indexed // by calling IndexPaths.match(path) ! if (idx == null ! || (idx.getIncludeAttributes() ! && idx.match(currentPath + "/@" + nodeName))) textEngine.storeAttribute(idx, (AttrImpl) node); // if the attribute has type ID, store the ID-value --- 1111,1124 ---- // check if attribute value should be fulltext-indexed // by calling IndexPaths.match(path) ! boolean indexAttribs = true; ! if(idx != null) { ! if(idx.getIncludeAttributes()) { ! currentPath.addComponent('@' + ((AttrImpl)node).getName()); ! indexAttribs = idx.match(currentPath); ! currentPath.removeLastComponent(); ! } else ! indexAttribs = false; ! } ! if (indexAttribs) textEngine.storeAttribute(idx, (AttrImpl) node); // if the attribute has type ID, store the ID-value *************** *** 1117,1121 **** // check if this textual content should be fulltext-indexed // by calling IndexPaths.match(path) ! if (idx == null || idx.match(currentPath.toString())) textEngine.storeText(idx, (TextImpl) node); break; --- 1133,1137 ---- // check if this textual content should be fulltext-indexed // by calling IndexPaths.match(path) ! if (idx == null || idx.match(currentPath)) textEngine.storeText(idx, (TextImpl) node); break; *************** *** 1132,1138 **** * @param currentPath */ ! private void scanNodes(Iterator iterator, NodeImpl node, StringBuffer currentPath) { if (node.getNodeType() == Node.ELEMENT_NODE) ! currentPath.append('/').append(node.getNodeName()); reindex(node, currentPath); if (node.hasChildNodes()) { --- 1148,1154 ---- * @param currentPath */ ! private void scanNodes(Iterator iterator, NodeImpl node, NodePath currentPath) { if (node.getNodeType() == Node.ELEMENT_NODE) ! currentPath.addComponent(node.getNodeName()); reindex(node, currentPath); if (node.hasChildNodes()) { *************** *** 1160,1163 **** --- 1176,1181 ---- } } + if(node.getNodeType() == Node.ELEMENT_NODE) + currentPath.removeLastComponent(); } *************** *** 1438,1442 **** lock.release(); } - elementPool.clear(); LOG.debug("removing dom nodes ..."); --- 1456,1459 ---- *************** *** 1597,1601 **** lock.release(); } - elementPool.clear(); ((NativeTextEngine) textEngine).removeDocument(doc); --- 1614,1617 ---- *************** *** 1925,1929 **** * fulltext-indexed). */ ! public void store(final NodeImpl node, CharSequence currentPath) { // first, check available memory if (nodesCount > MEM_LIMIT_CHECK) { --- 1941,1945 ---- * fulltext-indexed). */ ! public void store(final NodeImpl node, NodePath currentPath) { // first, check available memory if (nodesCount > MEM_LIMIT_CHECK) { *************** *** 1993,1999 **** // check if attribute value should be fulltext-indexed // by calling IndexPaths.match(path) ! if (idx == null ! || (idx.getIncludeAttributes() ! && idx.match(currentPath + "/@" + nodeName))) textEngine.storeAttribute(idx, (AttrImpl) node); // if the attribute has type ID, store the ID-value --- 2009,2022 ---- // check if attribute value should be fulltext-indexed // by calling IndexPaths.match(path) ! boolean indexAttribs = true; ! if(idx != null) { ! if(idx.getIncludeAttributes()) { ! currentPath.addComponent('@' + nodeName); ! indexAttribs = idx.match(currentPath); ! currentPath.removeLastComponent(); ! } else ! indexAttribs = false; ! } ! if(indexAttribs) textEngine.storeAttribute(idx, (AttrImpl) node); // if the attribute has type ID, store the ID-value Index: NativeElementIndex.java =================================================================== RCS file: /cvsroot/exist/eXist-1.0/src/org/exist/storage/NativeElementIndex.java,v retrieving revision 1.24 retrieving revision 1.25 diff -C2 -d -r1.24 -r1.25 *** NativeElementIndex.java 3 May 2004 13:08:43 -0000 1.24 --- NativeElementIndex.java 10 May 2004 11:22:41 -0000 1.25 *************** *** 28,32 **** import java.util.Map; ! import org.apache.log4j.Category; import org.dbxml.core.DBException; import org.dbxml.core.data.Value; --- 28,32 ---- import java.util.Map; ! import org.apache.log4j.Logger; import org.dbxml.core.DBException; import org.dbxml.core.data.Value; *************** *** 47,59 **** import org.exist.util.ProgressIndicator; import org.exist.util.ReadOnlyException; - //import org.exist.util.StorageAddress; public class NativeElementIndex extends ElementIndex { ! private static Category LOG = Category.getInstance(NativeElementIndex.class .getName()); - public final static int PARTITION_SIZE = 102400; - protected BFile dbElement; private VariableByteOutputStream os = new VariableByteOutputStream(); --- 47,56 ---- import org.exist.util.ProgressIndicator; import org.exist.util.ReadOnlyException; public class NativeElementIndex extends ElementIndex { ! private static Logger LOG = Logger.getLogger(NativeElementIndex.class .getName()); protected BFile dbElement; private VariableByteOutputStream os = new VariableByteOutputStream(); *************** *** 361,365 **** prevId = proxy.gid; os.writeLong(cid); - //os.writeFixedLong(proxy.getInternalAddress()); StorageAddress.write(proxy.getInternalAddress(), os); } --- 358,361 ---- Index: DBBroker.java =================================================================== RCS file: /cvsroot/exist/eXist-1.0/src/org/exist/storage/DBBroker.java,v retrieving revision 1.25 retrieving revision 1.26 diff -C2 -d -r1.25 -r1.26 *** DBBroker.java 3 May 2004 13:08:43 -0000 1.25 --- DBBroker.java 10 May 2004 11:22:41 -0000 1.26 *************** *** 60,64 **** * *@author Wolfgang Meier <wol...@ex...> - *@created 20. Mai 2002 */ public abstract class DBBroker extends Observable { --- 60,63 ---- *************** *** 80,90 **** --- 79,94 ---- protected Configuration config; + protected BrokerPool pool; + protected File symbolsFile; protected SymbolTable symbols = null; + protected User user = null; private int referenceCount = 0; + protected int xupdateGrowthFactor = 1; + protected void saveSymbols() throws EXistException { synchronized (symbols) { *************** *** 148,151 **** --- 152,157 ---- config.setProperty("db-connection.symbol-table", symbols); } + if ((xupdateGrowthFactor = config.getInteger("xupdate.growth-factor")) < 0) + xupdateGrowthFactor = 1; this.pool = pool; } *************** *** 479,483 **** * fulltext-indexed). */ ! public abstract void store(NodeImpl node, CharSequence currentPath); /** --- 485,489 ---- * fulltext-indexed). */ ! public abstract void store(NodeImpl node, NodePath currentPath); /** *************** *** 575,577 **** --- 581,587 ---- --referenceCount; } + + public int getXUpdateGrowthFactor() { + return xupdateGrowthFactor; + } } \ No newline at end of file Index: IndexPaths.java =================================================================== RCS file: /cvsroot/exist/eXist-1.0/src/org/exist/storage/IndexPaths.java,v retrieving revision 1.6 retrieving revision 1.7 diff -C2 -d -r1.6 -r1.7 *** IndexPaths.java 25 Nov 2003 12:54:15 -0000 1.6 --- IndexPaths.java 10 May 2004 11:22:41 -0000 1.7 *************** *** 1,5 **** /* * eXist Open Source Native XML Database ! * Copyright (C) 2001, Wolfgang Meier * * This program is free software; you can redistribute it and/or --- 1,5 ---- /* * eXist Open Source Native XML Database ! * Copyright (C) 2001-04, Wolfgang Meier * * This program is free software; you can redistribute it and/or *************** *** 17,21 **** * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * ! * $Id: */ package org.exist.storage; --- 17,21 ---- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * ! * $Id$ */ package org.exist.storage; *************** *** 25,31 **** import java.util.Iterator; - import org.exist.util.FastStringBuffer; - import org.exist.util.MutableStringTokenizer; - /** --- 25,28 ---- *************** *** 42,55 **** private final static HashMap cache = new HashMap(); - private final static int MAX_CACHE_SIZE = 64; protected ArrayList includePath; protected ArrayList excludePath; protected boolean includeByDefault = true; protected boolean includeAttributes = true; protected boolean includeAlphaNum = true; protected int depth = 1; - protected MutableStringTokenizer tokenizer = new MutableStringTokenizer(); - private FastStringBuffer token = new FastStringBuffer(); /** --- 39,51 ---- private final static HashMap cache = new HashMap(); protected ArrayList includePath; protected ArrayList excludePath; + protected boolean includeByDefault = true; protected boolean includeAttributes = true; protected boolean includeAlphaNum = true; + protected int depth = 1; /** *************** *** 71,78 **** */ public void addInclude( String path ) { ! ArrayList a = tokenize( path ); ! String arr[] = new String[a.size()]; ! arr = (String[])a.toArray(arr); ! includePath.add( arr ); } --- 67,71 ---- */ public void addInclude( String path ) { ! includePath.add( new NodePath(path) ); } *************** *** 83,90 **** */ public void addExclude( String path ) { ! ArrayList a = tokenize( path ); ! String arr[] = new String[a.size()]; ! arr = (String[])a.toArray(arr); ! excludePath.add( arr ); } --- 76,80 ---- */ public void addExclude( String path ) { ! excludePath.add( new NodePath(path) ); } *************** *** 153,230 **** * @return Description of the Return Value */ ! public boolean match( CharSequence path ) { if ( includeByDefault ) { // check exclusions for ( Iterator i = excludePath.iterator(); i.hasNext( ); ) ! if ( match( (String[]) i.next( ), path ) ) return false; ! return true; } for ( Iterator i = includePath.iterator(); i.hasNext(); ) ! if ( match( (String[]) i.next(), path ) ) return true; return false; } - - private final boolean match( String[] path, CharSequence other) { - int i = 0; - boolean skip = false; - tokenizer.set(other, "/"); - CharSequence next; - while((next = tokenizer.nextToken()) != null) { - if(i == path.length) - return true; - if(path[i].equals("*")) { - ++i; - skip = true; - } - if(next.equals(path[i])) { - ++i; - skip = false; - } else if(skip) { - continue; - } else - return false; - } - return i == path.length; - } - - /** - * Description of the Method - * - * @param path Description of the Parameter - * - * @return Description of the Return Value - */ - private final ArrayList tokenize( String path ) { - ArrayList temp = new ArrayList(); - String next; - token.reset(); - int pos = 0; - char ch = 0; - final int pathLen = path.length(); - while ( pos < pathLen ) { - ch = path.charAt(pos); - switch ( ch ) { - case '/': - next = token.toString(); - token.reset(); - if ( next.length( ) > 0 ) - temp.add( next ); - if ( path.charAt( ++pos ) == '/' ) - temp.add( "*" ); - break; - - default: - token.append(ch); - pos++; - } - } - if ( token.length() > 0 ) - temp.add( token.toString() ); - return temp; - } } --- 143,161 ---- * @return Description of the Return Value */ ! public boolean match( NodePath path ) { if ( includeByDefault ) { // check exclusions for ( Iterator i = excludePath.iterator(); i.hasNext( ); ) ! if( ((NodePath)i.next()).match(path) ) return false; ! return true; } for ( Iterator i = includePath.iterator(); i.hasNext(); ) ! if( ((NodePath)i.next()).match(path) ) return true; return false; } } |