From: <ust...@us...> - 2008-07-18 12:16:25
|
Revision: 2780 http://clucene.svn.sourceforge.net/clucene/?rev=2780&view=rev Author: ustramooner Date: 2008-07-18 12:16:18 +0000 (Fri, 18 Jul 2008) Log Message: ----------- various fixes for mingw32 and msvc 8 Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.cpp branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.h branches/lucene2_3_2/src/core/CLucene/index/SegmentInfos.cpp branches/lucene2_3_2/src/core/CLucene/store/FSDirectory.cpp branches/lucene2_3_2/src/core/CLucene/store/Lock.cpp branches/lucene2_3_2/src/core/CLucene/store/LockFactory.cpp branches/lucene2_3_2/src/core/CLucene/store/RAMDirectory.cpp branches/lucene2_3_2/src/core/CLucene/util/Equators.h branches/lucene2_3_2/src/demo/stdafx.h branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h branches/lucene2_3_2/src/shared/CLucene/clucene-config.h.cmake branches/lucene2_3_2/src/shared/CLucene/util/dirent.h branches/lucene2_3_2/src/shared/CMakeLists.txt branches/lucene2_3_2/src/shared/cmake/CheckNamespace.cmake branches/lucene2_3_2/src/test/CMakeLists.txt branches/lucene2_3_2/src/test/search/TestDateFilter.cpp branches/lucene2_3_2/src/test/store/TestStore.cpp branches/lucene2_3_2/src/test/test.h Modified: branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.cpp 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.cpp 2008-07-18 12:16:18 UTC (rev 2780) @@ -12,8 +12,6 @@ CL_NS_USE(util) CL_NS_DEF(analysis) -const TCHAR* Token::defaultType=_T("word"); - struct Analyzer::Internal{ CL_NS(util)::ThreadLocal<TokenStream*, CL_NS(util)::Deletor::Object<TokenStream> >* tokenStreams; @@ -52,7 +50,7 @@ Token::Token(): _startOffset (0), _endOffset (0), - _type ( defaultType ), + _type ( getDefaultType() ), positionIncrement (1), payload(NULL) { @@ -76,7 +74,7 @@ Token::Token(const TCHAR* text, const int32_t start, const int32_t end, const TCHAR* typ): _startOffset (start), _endOffset (end), - _type ( typ ), + _type ( (typ==NULL?getDefaultType():typ) ), positionIncrement (1), payload(NULL) { @@ -91,10 +89,14 @@ setText(text); } +const TCHAR* Token::getDefaultType(){ + return _T("word"); +} + void Token::set(const TCHAR* text, const int32_t start, const int32_t end, const TCHAR* typ){ _startOffset = start; _endOffset = end; - _type = typ; + _type = (typ==NULL?getDefaultType():typ); positionIncrement = 1; setText(text); } Modified: branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.h 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.h 2008-07-18 12:16:18 UTC (rev 2780) @@ -14,58 +14,58 @@ typedef CL_NS(util)::CLSetList<const TCHAR*, CL_NS(util)::Compare::TChar, CL_NS(util)::Deletor::tcArray> CLTCSetList; -/** A Token is an occurence of a term from the text of a field. It consists of - a term's text, the start and end offset of the term in the text of the field, - and a type string. - <p> - The start and end offsets permit applications to re-associate a token with - its source text, e.g., to display highlighted query terms in a document - browser, or to show matching text fragments in a KWIC (KeyWord In Context) - display, etc. - <p> - The type is an interned string, assigned by a lexical analyzer - (a.k.a. tokenizer), naming the lexical or syntactic class that the token - belongs to. For example an end of sentence marker token might be implemented - with type "eos". The default token type is "word". - <p> - A Token can optionally have metadata (a.k.a. Payload) in the form of a variable - length byte array. Use {@link TermPositions#getPayloadLength()} and - {@link TermPositions#getPayload(byte[], int)} to retrieve the payloads from the index. - - <br><br> - <p><font color="#FF0000"> - WARNING: The status of the <b>Payloads</b> feature is experimental. - The APIs introduced here might change in the future and will not be - supported anymore in such a case.</font> - - <br><br> - - <p><b>NOTE:</b> As of 2.3, Token stores the term text - internally as a malleable char[] termBuffer instead of - String termText. The indexing code and core tokenizers - have been changed re-use a single Token instance, changing - its buffer and other fields in-place as the Token is - processed. This provides substantially better indexing - performance as it saves the GC cost of new'ing a Token and - String for every term. The APIs that accept String - termText are still available but a warning about the - associated performance cost has been added (below). The - {@link #termText()} method has been deprecated.</p> - - <p>Tokenizers and filters should try to re-use a Token - instance when possible for best performance, by - implementing the {@link TokenStream#next(Token)} API. - Failing that, to create a new Token you should first use - one of the constructors that starts with null text. Then - you should call either {@link #termBuffer()} or {@link - #resizeTermBuffer(int)} to retrieve the Token's - termBuffer. Fill in the characters of your term into this - buffer, and finally call {@link #setTermLength(int)} to - set the length of the term text. See <a target="_top" - href="https://issues.apache.org/jira/browse/LUCENE-969">LUCENE-969</a> - for details.</p> - - @see Payload +/** A Token is an occurence of a term from the text of a field. It consists of + a term's text, the start and end offset of the term in the text of the field, + and a type string. + <p> + The start and end offsets permit applications to re-associate a token with + its source text, e.g., to display highlighted query terms in a document + browser, or to show matching text fragments in a KWIC (KeyWord In Context) + display, etc. + <p> + The type is an interned string, assigned by a lexical analyzer + (a.k.a. tokenizer), naming the lexical or syntactic class that the token + belongs to. For example an end of sentence marker token might be implemented + with type "eos". The default token type is "word". + <p> + A Token can optionally have metadata (a.k.a. Payload) in the form of a variable + length byte array. Use {@link TermPositions#getPayloadLength()} and + {@link TermPositions#getPayload(byte[], int)} to retrieve the payloads from the index. + + <br><br> + <p><font color="#FF0000"> + WARNING: The status of the <b>Payloads</b> feature is experimental. + The APIs introduced here might change in the future and will not be + supported anymore in such a case.</font> + + <br><br> + + <p><b>NOTE:</b> As of 2.3, Token stores the term text + internally as a malleable char[] termBuffer instead of + String termText. The indexing code and core tokenizers + have been changed re-use a single Token instance, changing + its buffer and other fields in-place as the Token is + processed. This provides substantially better indexing + performance as it saves the GC cost of new'ing a Token and + String for every term. The APIs that accept String + termText are still available but a warning about the + associated performance cost has been added (below). The + {@link #termText()} method has been deprecated.</p> + + <p>Tokenizers and filters should try to re-use a Token + instance when possible for best performance, by + implementing the {@link TokenStream#next(Token)} API. + Failing that, to create a new Token you should first use + one of the constructors that starts with null text. Then + you should call either {@link #termBuffer()} or {@link + #resizeTermBuffer(int)} to retrieve the Token's + termBuffer. Fill in the characters of your term into this + buffer, and finally call {@link #setTermLength(int)} to + set the length of the term text. See <a target="_top" + href="https://issues.apache.org/jira/browse/LUCENE-969">LUCENE-969</a> + for details.</p> + + @see Payload */ class CLUCENE_EXPORT Token:LUCENE_BASE{ private: @@ -82,15 +82,15 @@ TCHAR _termText[LUCENE_TOKEN_WORD_LENGTH+1]; ///< the text of the term #endif int32_t _termTextLen; ///< the length of termText. Internal use only - static const TCHAR* defaultType; + static const TCHAR* getDefaultType(); CL_NS(index)::Payload* payload; Token(); ~Token(); /// Constructs a Token with the given text, start and end offsets, & type. - Token(const TCHAR* text, const int32_t start, const int32_t end, const TCHAR* typ=defaultType); - void set(const TCHAR* text, const int32_t start, const int32_t end, const TCHAR* typ=defaultType); + Token(const TCHAR* text, const int32_t start, const int32_t end, const TCHAR* typ=NULL); + void set(const TCHAR* text, const int32_t start, const int32_t end, const TCHAR* typ=NULL); size_t bufferLength(){ return bufferTextLen; } void growBuffer(size_t size); @@ -124,13 +124,13 @@ void resetTermTextLen(); void setText(const TCHAR* txt); - /** Returns the internal termBuffer character array which - * you can then directly alter. If the array is too - * small for your token, use {@link - * #resizeTermBuffer(int)} to increase it. After - * altering the buffer be sure to call {@link - * #setTermLength} to record the number of valid - * characters that were placed into the termBuffer. */ + /** Returns the internal termBuffer character array which + * you can then directly alter. If the array is too + * small for your token, use {@link + * #resizeTermBuffer(int)} to increase it. After + * altering the buffer be sure to call {@link + * #setTermLength} to record the number of valid + * characters that were placed into the termBuffer. */ const TCHAR* termBuffer() const; /** @@ -143,7 +143,7 @@ */ int32_t startOffset() const { return _startOffset; } - /** Set the starting offset. + /** Set the starting offset. @see #startOffset() */ void setStartOffset(const int32_t val){ _startOffset = val; } @@ -153,7 +153,7 @@ */ int32_t endOffset() const { return _endOffset; } - /** Set the ending offset. + /** Set the ending offset. @see #endOffset() */ void setEndOffset(const int32_t val){ _endOffset = val; } @@ -161,36 +161,36 @@ const TCHAR* type() const { return _type; } ///<returns reference void setType(const TCHAR* val) { _type = val; } ///<returns reference - /** - * Returns this Token's payload. - */ - CL_NS(index)::Payload* getPayload(); - - /** - * Sets this Token's payload. - */ + /** + * Returns this Token's payload. + */ + CL_NS(index)::Payload* getPayload(); + + /** + * Sets this Token's payload. + */ void setPayload(CL_NS(index)::Payload* payload); - /** Resets the term text, payload, and positionIncrement to default. - * Other fields such as startOffset, endOffset and the token type are - * not reset since they are normally overwritten by the tokenizer. */ - void clear(); + /** Resets the term text, payload, and positionIncrement to default. + * Other fields such as startOffset, endOffset and the token type are + * not reset since they are normally overwritten by the tokenizer. */ + void clear(); TCHAR* toString() const; }; -/** A TokenStream enumerates the sequence of tokens, either from -fields of a document or from query text. -<p> -This is an abstract class. Concrete subclasses are: -<ul> -<li>{@link Tokenizer}, a TokenStream -whose input is a Reader; and -<li>{@link TokenFilter}, a TokenStream -whose input is another TokenStream. -</ul> -NOTE: subclasses must override at least one of {@link -#next()} or {@link #next(Token)}. +/** A TokenStream enumerates the sequence of tokens, either from +fields of a document or from query text. +<p> +This is an abstract class. Concrete subclasses are: +<ul> +<li>{@link Tokenizer}, a TokenStream +whose input is a Reader; and +<li>{@link TokenFilter}, a TokenStream +whose input is another TokenStream. +</ul> +NOTE: subclasses must override at least one of {@link +#next()} or {@link #next(Token)}. */ class CLUCENE_EXPORT TokenStream:LUCENE_BASE { public: @@ -206,13 +206,13 @@ /** Releases resources associated with this stream. */ virtual void close() = 0; - /** Resets this stream to the beginning. This is an - * optional operation, so subclasses may or may not - * implement this method. Reset() is not needed for - * the standard indexing process. However, if the Tokens - * of a TokenStream are intended to be consumed more than - * once, it is necessary to implement reset(). - */ + /** Resets this stream to the beginning. This is an + * optional operation, so subclasses may or may not + * implement this method. Reset() is not needed for + * the standard indexing process. However, if the Tokens + * of a TokenStream are intended to be consumed more than + * once, it is necessary to implement reset(). + */ //virtual void reset(CL_NS(util)::Reader* _input=NULL) = 0; virtual ~TokenStream(){ @@ -287,12 +287,12 @@ /** A Tokenizer is a TokenStream whose input is a Reader. <p> This is an abstract class. -<p> -NOTE: subclasses must override at least one of {@link -#next()} or {@link #next(Token)}. -<p> -NOTE: subclasses overriding {@link #next(Token)} must -call {@link Token#clear()}. +<p> +NOTE: subclasses must override at least one of {@link +#next()} or {@link #next(Token)}. +<p> +NOTE: subclasses overriding {@link #next(Token)} must +call {@link Token#clear()}. */ class CLUCENE_EXPORT Tokenizer:public TokenStream { protected: Modified: branches/lucene2_3_2/src/core/CLucene/index/SegmentInfos.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/index/SegmentInfos.cpp 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/core/CLucene/index/SegmentInfos.cpp 2008-07-18 12:16:18 UTC (rev 2780) @@ -15,11 +15,7 @@ //#include "CLucene/util/VoidMap.h" #include "CLucene/util/Misc.h" -#ifdef _CL_HAVE_WINDEF_H - #include <windef.h> -#endif - CL_NS_USE(store) CL_NS_USE(util) CL_NS_DEF(index) Modified: branches/lucene2_3_2/src/core/CLucene/store/FSDirectory.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/store/FSDirectory.cpp 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/core/CLucene/store/FSDirectory.cpp 2008-07-18 12:16:18 UTC (rev 2780) @@ -6,9 +6,6 @@ ------------------------------------------------------------------------------*/ #include "CLucene/_ApiHeader.h" -#ifdef _CL_HAVE_WINDEF_H - #include <windef.h> //try to include windef.h as early as possible -#endif #include <fcntl.h> #ifdef _CL_HAVE_IO_H #include <io.h> @@ -22,9 +19,6 @@ #ifdef _CL_HAVE_DIRECT_H #include <direct.h> #endif -#ifdef _CL_HAVE_WINDEF_H - #include <windef.h> -#endif #include <errno.h> #include "FSDirectory.h" Modified: branches/lucene2_3_2/src/core/CLucene/store/Lock.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/store/Lock.cpp 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/core/CLucene/store/Lock.cpp 2008-07-18 12:16:18 UTC (rev 2780) @@ -9,9 +9,6 @@ #include "_Lock.h" #include "CLucene/util/Misc.h" -#ifdef _CL_HAVE_WINDEF_H - #include <windef.h> //try to include windef.h as early as possible -#endif #ifdef _CL_HAVE_IO_H #include <io.h> #endif @@ -147,6 +144,7 @@ TCHAR* FSLock::toString() { + const wchar_t* xx= L"SimpleFSLock@"; TCHAR* ret = _CL_NEWARRAY(TCHAR,strlen(lockFile)+14); // 14 = strlen("SimpleFSLock@") _tcscpy(ret,_T("SimpleFSLock@")); STRCPY_AtoT(ret+13,lockFile,strlen(lockFile)+1); Modified: branches/lucene2_3_2/src/core/CLucene/store/LockFactory.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/store/LockFactory.cpp 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/core/CLucene/store/LockFactory.cpp 2008-07-18 12:16:18 UTC (rev 2780) @@ -10,9 +10,6 @@ #include "_Lock.h" #include "CLucene/util/Misc.h" -#ifdef _CL_HAVE_WINDEF_H - #include <windef.h> //try to include windef.h as early as possible -#endif #ifdef _CL_HAVE_SYS_STAT_H #include <sys/stat.h> #endif Modified: branches/lucene2_3_2/src/core/CLucene/store/RAMDirectory.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/store/RAMDirectory.cpp 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/core/CLucene/store/RAMDirectory.cpp 2008-07-18 12:16:18 UTC (rev 2780) @@ -15,10 +15,6 @@ //#include "CLucene/util/VoidMap.h" #include "CLucene/util/Misc.h" -#ifdef _CL_HAVE_WINDEF_H - #include <windef.h> -#endif - CL_NS_USE(util) CL_NS_DEF(store) Modified: branches/lucene2_3_2/src/core/CLucene/util/Equators.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/Equators.h 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/core/CLucene/util/Equators.h 2008-07-18 12:16:18 UTC (rev 2780) @@ -11,9 +11,6 @@ #include <list> #include <set> #include <vector> -#ifdef HAVE_WINDOWS_H - #include <windef.h> -#endif //#include "CLucene/LuceneThreads.h" CL_NS_DEF(util) Modified: branches/lucene2_3_2/src/demo/stdafx.h =================================================================== --- branches/lucene2_3_2/src/demo/stdafx.h 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/demo/stdafx.h 2008-07-18 12:16:18 UTC (rev 2780) @@ -13,7 +13,6 @@ #define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers #include <stdio.h> -//#include <tchar.h> #include "CLucene/StdHeader.h" #include "CLucene/_clucene-config.h" Modified: branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-07-18 12:16:18 UTC (rev 2780) @@ -17,6 +17,17 @@ #include "CLucene/clucene-config.h" +//some early definitions +#if defined(_MSC_VER) + //this has to be included very early, or visual studio will ignore it + #pragma warning(disable : 4786) //ignore: identifier was truncated to '255' characters in the debug information + #define _LUCENE_PRAGMA_WARNINGS //tell lucene to display warnings using pragmas instead of #warning +#endif +#if defined(__BORLANDC__) + #define _LUCENE_PRAGMA_WARNINGS //tell lucene to display warnings using pragmas instead of #warning +#endif + + //////////////////////////////////////////////////////// //Are we in unicode mode? //////////////////////////////////////////////////////// @@ -144,16 +155,7 @@ #endif #define _CL_DEPRECATED(_NewItem) _CL_DEPRECATE_TEXT("This function or variable has been superceded by newer library or operating system functionality. Consider using " #_NewItem " instead. See online help for details.") -#if defined(_MSC_VER) - //this has to be included very early, or visual studio will ignore it - #pragma warning(disable : 4786) //ignore: identifier was truncated to '255' characters in the debug information - #define _LUCENE_PRAGMA_WARNINGS //tell lucene to display warnings using pragmas instead of #warning -#endif -#if defined(__BORLANDC__) - #define _LUCENE_PRAGMA_WARNINGS //tell lucene to display warnings using pragmas instead of #warning -#endif - //////////////////////////////////////////////////////// //Class interfaces //////////////////////////////////////////////////////// Modified: branches/lucene2_3_2/src/shared/CLucene/clucene-config.h.cmake =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/clucene-config.h.cmake 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/shared/CLucene/clucene-config.h.cmake 2008-07-18 12:16:18 UTC (rev 2780) @@ -46,7 +46,8 @@ ${TYPE_FLOAT_T} ${TYPE_SIZE_T} -${SYMBOL_TCHAR} +/* tchar & _T definitions... */ +${TYPE_TCHAR} ${SYMBOL__T} /* CMake will determine these specifics. Things like bugs, etc */ Modified: branches/lucene2_3_2/src/shared/CLucene/util/dirent.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/util/dirent.h 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/shared/CLucene/util/dirent.h 2008-07-18 12:16:18 UTC (rev 2780) @@ -13,11 +13,7 @@ typedef int intptr_t; #include <io.h> -#ifdef _CL_HAVE_WINDEF_H - #include <windef.h> -#endif - /** \unit * dirent.c Modified: branches/lucene2_3_2/src/shared/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-07-18 12:16:18 UTC (rev 2780) @@ -175,11 +175,11 @@ CHOOSE_SYMBOL (_T "_T" SYMBOL__T) IF ( NOT HAVE_SYMBOL__T ) - IF ( ENABLE_ASCII_MODE ) - SET (SYMBOL__T "#define _T(x) x") - ELSE ( ENABLE_ASCII_MODE ) - SET (SYMBOL__T "#define _T(x) L ## x") - ENDIF ( ENABLE_ASCII_MODE ) +IF ( ENABLE_ASCII_MODE ) + SET (SYMBOL__T "#define _T(x) x") +ELSE ( ENABLE_ASCII_MODE ) + SET (SYMBOL__T "#define _T(x) L ## x") +ENDIF ( ENABLE_ASCII_MODE ) ELSE ( NOT HAVE_SYMBOL__T ) SET( SYMBOL__T "/* #undef _T */" ) ENDIF ( NOT HAVE_SYMBOL__T ) Modified: branches/lucene2_3_2/src/shared/cmake/CheckNamespace.cmake =================================================================== --- branches/lucene2_3_2/src/shared/cmake/CheckNamespace.cmake 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/shared/cmake/CheckNamespace.cmake 2008-07-18 12:16:18 UTC (rev 2780) @@ -1,7 +1,7 @@ #check if we support namespaces MACRO ( CHECK_NAMESPACE haveNamespace ) #Check if namespaces work in the compiler - CHECK_CXX_SOURCE_COMPILES(" + CHECK_CXX_SOURCE_RUNS(" namespace Outer { namespace Inner { int i = 0; }} int main(){ using namespace Outer::Inner; return i; }" ${haveNamespace} ) Modified: branches/lucene2_3_2/src/test/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/test/CMakeLists.txt 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/test/CMakeLists.txt 2008-07-18 12:16:18 UTC (rev 2780) @@ -42,7 +42,8 @@ ./util/English.cpp ${test_HEADERS} ) - + +#todo: do glob header and include header files for IDE. ADD_EXECUTABLE(cl_test ${test_files} ) SET_TARGET_PROPERTIES(cl_test PROPERTIES DEFINE_SYMBOL "") Modified: branches/lucene2_3_2/src/test/search/TestDateFilter.cpp =================================================================== --- branches/lucene2_3_2/src/test/search/TestDateFilter.cpp 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/test/search/TestDateFilter.cpp 2008-07-18 12:16:18 UTC (rev 2780) @@ -5,9 +5,6 @@ * the GNU Lesser General Public License, as specified in the COPYING file. ------------------------------------------------------------------------------*/ #include "test.h" -#ifdef _CL_HAVE_WINDOWS_H - #include <windows.h> -#endif void testBefore(CuTest *tc) { // create an index Modified: branches/lucene2_3_2/src/test/store/TestStore.cpp =================================================================== --- branches/lucene2_3_2/src/test/store/TestStore.cpp 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/test/store/TestStore.cpp 2008-07-18 12:16:18 UTC (rev 2780) @@ -8,13 +8,9 @@ #include "CLucene/store/Directory.h" #include "CLucene/store/IndexInput.h" -#ifdef _CL_HAVE_WINDOWS_H - #include <windows.h> -#endif - -//todo: this is testing internal stuff void hashTest(CuTest *tc){ - CLHashMap<const char*,int,Compare::Char,Equals::Char,Deletor::acArray,Deletor::ConstNullVal<int> > map(true,true); + //todo: this is testing internal stuff, turn it on again using a shared test.. + /*CLHashMap<const char*,int,Compare::Char,Equals::Char,Deletor::acArray,Deletor::ConstNullVal<int> > map(true,true); map.put(STRDUP_AtoA("a1"),1); map.put(STRDUP_AtoA("a1"),2); map.put(STRDUP_AtoA("a2"),3); @@ -27,7 +23,7 @@ CuAssertIntEquals(tc, _T("map size!=2"), 2, map.size()); map.put(STRDUP_AtoA("a1"),5); - CuAssertIntEquals(tc, _T("mapsize!=3"),3, map.size()); + CuAssertIntEquals(tc, _T("mapsize!=3"),3, map.size());*/ } void StoreTest(CuTest *tc,int32_t count, bool ram){ Modified: branches/lucene2_3_2/src/test/test.h =================================================================== --- branches/lucene2_3_2/src/test/test.h 2008-07-16 11:06:46 UTC (rev 2779) +++ branches/lucene2_3_2/src/test/test.h 2008-07-18 12:16:18 UTC (rev 2780) @@ -67,14 +67,35 @@ return ret; } }; +class CharCompare{ +public: + bool operator()( const char* val1, const char* val2 ) const{ + if ( val1==val2) + return false; + bool ret = (strcmp( val1,val2 ) < 0); + return ret; + } +}; -template<typename _K, typename _T> -class StringMap : public std::map<_K,_T,TCharCompare>{ +template<typename _K, typename _T, typename _Comparator=TCharCompare> +class StringMap : public std::map<_K,_T,_Comparator>{ bool delKey; public: StringMap(bool delKey){ this->delKey = delKey; } + void remove(_K val){ + std::iterator<_K,_T,_Comparator> itr; + itr = this->find(val); + if ( itr == this->end() ) + return; + _K v = itr->first; + this->erase(itr); + if ( delKey ){ + _CLDELETE_CARRAY(v); + } + } + virtual ~StringMap(){ while ( this->begin() != this->end() ){ _K v = this->begin()->first; This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-07-21 13:54:13
|
Revision: 2783 http://clucene.svn.sourceforge.net/clucene/?rev=2783&view=rev Author: ustramooner Date: 2008-07-21 13:54:09 +0000 (Mon, 21 Jul 2008) Log Message: ----------- solaris changes Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/search/FieldDocSortedHitQueue.cpp branches/lucene2_3_2/src/core/CMakeLists.txt branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h branches/lucene2_3_2/src/shared/CLucene/debug/mem.h branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp branches/lucene2_3_2/src/shared/CLucene/util/StringBuffer.cpp Modified: branches/lucene2_3_2/src/core/CLucene/search/FieldDocSortedHitQueue.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/search/FieldDocSortedHitQueue.cpp 2008-07-21 10:07:43 UTC (rev 2782) +++ branches/lucene2_3_2/src/core/CLucene/search/FieldDocSortedHitQueue.cpp 2008-07-21 13:54:09 UTC (rev 2783) @@ -55,21 +55,21 @@ if (fields[i]->getReverse()) { switch (type) { case SortField::DOCSCORE: - r1 = __REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); - r2 = __REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); + r1 = _CL_REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); + r2 = _CL_REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); if (r1 < r2) c = -1; if (r1 > r2) c = 1; break; case SortField::DOC: case SortField::INT: - i1 = __REINTERPRET_CAST(Compare::Int32*, docA->fields[i])->getValue(); - i2 = __REINTERPRET_CAST(Compare::Int32*, docB->fields[i])->getValue(); + i1 = _CL_REINTERPRET_CAST(Compare::Int32*, docA->fields[i])->getValue(); + i2 = _CL_REINTERPRET_CAST(Compare::Int32*, docB->fields[i])->getValue(); if (i1 > i2) c = -1; if (i1 < i2) c = 1; break; case SortField::STRING: - s1 = __REINTERPRET_CAST(Compare::TChar*, docA->fields[i])->getValue(); - s2 = __REINTERPRET_CAST(Compare::TChar*, docB->fields[i])->getValue(); + s1 = _CL_REINTERPRET_CAST(Compare::TChar*, docA->fields[i])->getValue(); + s2 = _CL_REINTERPRET_CAST(Compare::TChar*, docB->fields[i])->getValue(); if (s2 == NULL) c = -1; // could be NULL if there are else if (s1 == NULL) c = 1; // no terms in the given field else c = _tcscmp(s2,s1); //else if (fields[i].getLocale() == NULL) { @@ -80,8 +80,8 @@ }*/ break; case SortField::FLOAT: - f1 = __REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); - f2 = __REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); + f1 = _CL_REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); + f2 = _CL_REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); if (f1 > f2) c = -1; if (f1 < f2) c = 1; break; @@ -100,21 +100,21 @@ } else { switch (type) { case SortField::DOCSCORE: - r1 = __REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); - r2 = __REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); + r1 = _CL_REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); + r2 = _CL_REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); if (r1 > r2) c = -1; if (r1 < r2) c = 1; break; case SortField::DOC: case SortField::INT: - i1 = __REINTERPRET_CAST(Compare::Int32*, docA->fields[i])->getValue(); - i2 = __REINTERPRET_CAST(Compare::Int32*, docB->fields[i])->getValue(); + i1 = _CL_REINTERPRET_CAST(Compare::Int32*, docA->fields[i])->getValue(); + i2 = _CL_REINTERPRET_CAST(Compare::Int32*, docB->fields[i])->getValue(); if (i1 < i2) c = -1; if (i1 > i2) c = 1; break; case SortField::STRING: - s1 = __REINTERPRET_CAST(Compare::TChar*, docA->fields[i])->getValue(); - s2 = __REINTERPRET_CAST(Compare::TChar*, docB->fields[i])->getValue(); + s1 = _CL_REINTERPRET_CAST(Compare::TChar*, docA->fields[i])->getValue(); + s2 = _CL_REINTERPRET_CAST(Compare::TChar*, docB->fields[i])->getValue(); // NULL values need to be sorted first, because of how FieldCache.getStringIndex() // works - in that routine, any documents without a value in the given field are // put first. @@ -127,8 +127,8 @@ }*/ break; case SortField::FLOAT: - f1 = __REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); - f2 = __REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); + f1 = _CL_REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); + f2 = _CL_REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); if (f1 < f2) c = -1; if (f1 > f2) c = 1; break; Modified: branches/lucene2_3_2/src/core/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/core/CMakeLists.txt 2008-07-21 10:07:43 UTC (rev 2782) +++ branches/lucene2_3_2/src/core/CMakeLists.txt 2008-07-21 13:54:09 UTC (rev 2783) @@ -150,19 +150,13 @@ ) #install public headers. -#foreach(file ${HEADERS}) -# get_filename_component(apath ${file} PATH) -# get_filename_component(aname ${file} NAME) -# file(RELATIVE_PATH relpath ${CMAKE_SOURCE_DIR}/src ${apath}) -# -# IF (NOT relpath STREQUAL "demo" ) -# IF ( NOT aname MATCHES "^_.*" ) -# install(FILES ${file} DESTINATION include/${relpath} ) -# ENDIF ( NOT aname MATCHES "^_.*" ) -# ENDIF (NOT relpath STREQUAL "demo" ) -#endforeach(file) +FOREACH(file ${HEADERS}) + get_filename_component(apath ${file} PATH) + get_filename_component(aname ${file} NAME) + file(RELATIVE_PATH relpath ${CMAKE_SOURCE_DIR}/src/core ${apath}) +ENDFOREACH(file) #also install CLucene.h -#install(FILES ${clucene-core_BINARY_DIR}/clucene-config.h DESTINATION lib/CLucene ) +install(FILES ${clucene-shared_BINARY_DIR}/clucene-config.h DESTINATION lib/CLucene ) install(FILES ${HEADERS} DESTINATION include/) Modified: branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-07-21 10:07:43 UTC (rev 2782) +++ branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-07-21 13:54:09 UTC (rev 2783) @@ -138,8 +138,8 @@ //////////////////////////////////////////////////////// -#define __CONST_CAST(typ,var) const_cast<typ>(var) -#define __REINTERPRET_CAST(typ,var) reinterpret_cast<typ>(var) +#define _CL_CONST_CAST(typ,var) const_cast<typ>(var) +#define _CL_REINTERPRET_CAST(typ,var) reinterpret_cast<typ>(var) //todo: put this logic in cmake #if _MSC_FULL_VER >= 140050320 Modified: branches/lucene2_3_2/src/shared/CLucene/debug/mem.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/debug/mem.h 2008-07-21 10:07:43 UTC (rev 2782) +++ branches/lucene2_3_2/src/shared/CLucene/debug/mem.h 2008-07-21 13:54:09 UTC (rev 2783) @@ -29,10 +29,10 @@ #if (_MSC_VER < 1300) //6.0 - #define _CLDELETE_CARRAY(x) if (x!=NULL){delete[] __CONST_CAST(TCHAR*,x); x=NULL;} - #define _CLDELETE_CaARRAY(x) if (x!=NULL){delete[] __CONST_CAST(char*,x); x=NULL;} - #define _CLDELETE_LCARRAY(x) if (x!=NULL){delete[] __CONST_CAST(TCHAR*,x);} - #define _CLDELETE_LCaARRAY(x) if (x!=NULL){delete[] __CONST_CAST(char*,x);} + #define _CLDELETE_CARRAY(x) if (x!=NULL){delete[] _CL_CONST_CAST(TCHAR*,x); x=NULL;} + #define _CLDELETE_CaARRAY(x) if (x!=NULL){delete[] _CL_CONST_CAST(char*,x); x=NULL;} + #define _CLDELETE_LCARRAY(x) if (x!=NULL){delete[] _CL_CONST_CAST(TCHAR*,x);} + #define _CLDELETE_LCaARRAY(x) if (x!=NULL){delete[] _CL_CONST_CAST(char*,x);} #endif //Macro for creating new arrays Modified: branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp 2008-07-21 10:07:43 UTC (rev 2782) +++ branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp 2008-07-21 13:54:09 UTC (rev 2783) @@ -25,6 +25,9 @@ #ifdef _CL_HAVE_STRINGS_H #include <strings.h> #endif +#ifdef _CL_HAVE_UNISTD_H + #include <unistd.h> +#endif #include <cctype> #include <limits.h> Modified: branches/lucene2_3_2/src/shared/CLucene/util/StringBuffer.cpp =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/util/StringBuffer.cpp 2008-07-21 10:07:43 UTC (rev 2782) +++ branches/lucene2_3_2/src/shared/CLucene/util/StringBuffer.cpp 2008-07-21 13:54:09 UTC (rev 2783) @@ -165,8 +165,8 @@ int64_t v = (int64_t)value; //the integer value of the float _i64tot(v,buf,10); //add the whole number - size_t len = 99-_tcslen(buf); //how many digits we have to work with? - size_t dig = len< (size_t)digits ? len : digits; + size_t l = 99-_tcslen(buf); //how many digits we have to work with? + size_t dig = l< (size_t)digits ? l : digits; if ( dig > 0 ){ _tcscat(buf,_T(".")); //add a decimal point This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-07-21 16:27:22
|
Revision: 2788 http://clucene.svn.sourceforge.net/clucene/?rev=2788&view=rev Author: ustramooner Date: 2008-07-21 16:27:18 +0000 (Mon, 21 Jul 2008) Log Message: ----------- do stdcall check on several functions. this was tripping up sun and cygwin Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/StdHeader.cpp branches/lucene2_3_2/src/core/CLucene/search/FieldCache.h branches/lucene2_3_2/src/core/CLucene/search/FieldCacheImpl.cpp branches/lucene2_3_2/src/core/CLucene/store/MMapInput.cpp branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.h branches/lucene2_3_2/src/core/CLucene/util/inputstreambuffer.h branches/lucene2_3_2/src/shared/CLucene/_SharedHeader.h branches/lucene2_3_2/src/shared/CLucene/_clucene-config.h.cmake branches/lucene2_3_2/src/shared/CLucene/config/repl_wchar.h branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp branches/lucene2_3_2/src/shared/CMakeLists.txt Modified: branches/lucene2_3_2/src/core/CLucene/StdHeader.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/StdHeader.cpp 2008-07-21 14:57:07 UTC (rev 2787) +++ branches/lucene2_3_2/src/core/CLucene/StdHeader.cpp 2008-07-21 16:27:18 UTC (rev 2788) @@ -16,7 +16,6 @@ #if defined(_MSC_VER) && defined(_DEBUG) #define CRTDBG_MAP_ALLOC - #include <stdlib.h> #include <crtdbg.h> #endif Modified: branches/lucene2_3_2/src/core/CLucene/search/FieldCache.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/search/FieldCache.h 2008-07-21 14:57:07 UTC (rev 2787) +++ branches/lucene2_3_2/src/core/CLucene/search/FieldCache.h 2008-07-21 16:27:18 UTC (rev 2788) @@ -42,19 +42,8 @@ /** Creates one of these objects Consumes all memory given. */ - StringIndex (int32_t* values, TCHAR** lookup, int count) { - this->count = count; - this->order = values; - this->lookup = lookup; - } - - ~StringIndex(){ - _CLDELETE_ARRAY(order); - - for ( int i=0;i<count;i++ ) - _CLDELETE_CARRAY(lookup[i]); - _CLDELETE_ARRAY(lookup); - } + StringIndex (int32_t* values, TCHAR** lookup, int count); + ~StringIndex(); }; Modified: branches/lucene2_3_2/src/core/CLucene/search/FieldCacheImpl.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/search/FieldCacheImpl.cpp 2008-07-21 14:57:07 UTC (rev 2787) +++ branches/lucene2_3_2/src/core/CLucene/search/FieldCacheImpl.cpp 2008-07-21 16:27:18 UTC (rev 2788) @@ -62,7 +62,20 @@ } }; +FieldCache::StringIndex::StringIndex (int32_t* values, TCHAR** lookup, int count) { + this->count = count; + this->order = values; + this->lookup = lookup; +} +FieldCache::StringIndex::~StringIndex(){ + _CLDELETE_ARRAY(order); + + for ( int i=0;i<count;i++ ) + _CLDELETE_CARRAY(lookup[i]); + _CLDELETE_ARRAY(lookup); +} + FieldCacheImpl::FieldCacheImpl() { cache = _CLNEW fieldcacheCacheType(false,true); Modified: branches/lucene2_3_2/src/core/CLucene/store/MMapInput.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/store/MMapInput.cpp 2008-07-21 14:57:07 UTC (rev 2787) +++ branches/lucene2_3_2/src/core/CLucene/store/MMapInput.cpp 2008-07-21 16:27:18 UTC (rev 2788) @@ -141,7 +141,7 @@ char* lpMsgBuf=0; _cl_dword_t dw = GetLastError(); - /*FormatMessageA( + FormatMessageA( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, @@ -154,7 +154,7 @@ sprintf(errstr, "MMapIndexInput::MMapIndexInput failed with error %d: %s", dw, lpMsgBuf); LocalFree(lpMsgBuf); - _CLTHROWA_DEL(CL_ERR_IO,errstr);*/ + _CLTHROWA_DEL(CL_ERR_IO,errstr); } #else //_CL_HAVE_FUNCTION_MAPVIEWOFFILE Modified: branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.h 2008-07-21 14:57:07 UTC (rev 2787) +++ branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.h 2008-07-21 16:27:18 UTC (rev 2788) @@ -16,7 +16,6 @@ #ifndef FILEINPUTSTREAM_H #define FILEINPUTSTREAM_H -#include <stdio.h> #include "bufferedstream.h" namespace jstreams { Modified: branches/lucene2_3_2/src/core/CLucene/util/inputstreambuffer.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/inputstreambuffer.h 2008-07-21 14:57:07 UTC (rev 2787) +++ branches/lucene2_3_2/src/core/CLucene/util/inputstreambuffer.h 2008-07-21 16:27:18 UTC (rev 2788) @@ -26,8 +26,6 @@ #ifndef INPUTSTREAMBUFFER_H #define INPUTSTREAMBUFFER_H -#include <cstdlib> - namespace jstreams { template <class T> Modified: branches/lucene2_3_2/src/shared/CLucene/_SharedHeader.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/_SharedHeader.h 2008-07-21 14:57:07 UTC (rev 2787) +++ branches/lucene2_3_2/src/shared/CLucene/_SharedHeader.h 2008-07-21 16:27:18 UTC (rev 2788) @@ -12,13 +12,16 @@ #include "CLucene/_clucene-config.h" #include "CLucene/SharedHeader.h" +//required globally (internally only) +#include <stdio.h> +#include <stdlib.h> + //we always need this stuff.... #include "CLucene/debug/_condition.h" #include "CLucene/LuceneThreads.h" #include "CLucene/config/repl_tchar.h" #include "CLucene/config/repl_wchar.h" - #define cl_min(a,b) (a>b?b:a) #define cl_max(a,b) (a>b?a:b) Modified: branches/lucene2_3_2/src/shared/CLucene/_clucene-config.h.cmake =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/_clucene-config.h.cmake 2008-07-21 14:57:07 UTC (rev 2787) +++ branches/lucene2_3_2/src/shared/CLucene/_clucene-config.h.cmake 2008-07-21 16:27:18 UTC (rev 2788) @@ -93,17 +93,11 @@ /* Compiler oddities */ -//not sure why, but cygwin reports _mkdir, but doesn't actually work... +//not sure why, but cygwin reports _S_IREAD, but doesn't actually work... //TODO: make this work properly (this bit shouldn't be necessary) #ifdef __CYGWIN__ - #define _mkdir(x) mkdir(x,0777) - #define _open open - #define _read read - #define _write write - #define _close close - #define _unlink unlink #define _ftime ftime - + #define _S_IREAD 0333 #define _S_IWRITE 0333 #endif Modified: branches/lucene2_3_2/src/shared/CLucene/config/repl_wchar.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/config/repl_wchar.h 2008-07-21 14:57:07 UTC (rev 2787) +++ branches/lucene2_3_2/src/shared/CLucene/config/repl_wchar.h 2008-07-21 16:27:18 UTC (rev 2788) @@ -7,7 +7,13 @@ #ifndef _lucene_repl_wchar_h #define _lucene_repl_wchar_h -#include <cstdarg> +#include <stdarg.h> +#ifdef _CL_HAVE_STRING_H + #include <string.h> +#endif +#ifdef _CL_HAVE_WCHAR_H + #include <wchar.h> +#endif int cl_tcscasefoldcmp(const TCHAR * dst, const TCHAR * src); Modified: branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp 2008-07-21 14:57:07 UTC (rev 2787) +++ branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp 2008-07-21 16:27:18 UTC (rev 2788) @@ -7,9 +7,6 @@ #include "CLucene/_ApiHeader.h" #include "Misc.h" -#include <stdio.h> -#include <stdlib.h> - #if defined(_CL_HAVE_SYS_TIME_H) # include <sys/time.h> #elif defined(_CL_HAVE_TIME_H) Modified: branches/lucene2_3_2/src/shared/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-07-21 14:57:07 UTC (rev 2787) +++ branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-07-21 16:27:18 UTC (rev 2788) @@ -40,7 +40,7 @@ CHECK_INCLUDE_FILES ("sys/time.h;time.h" _CL_TIME_WITH_SYS_TIME) -CHECK_REQUIRED_HEADERS ( stdlib.h stdarg.h stddef.h ctype.h algorithm +CHECK_REQUIRED_HEADERS ( stdlib.h stdarg.h stdio.h stddef.h ctype.h algorithm functional map vector list set math.h fcntl.h limits.h) CHECK_OPTIONAL_HEADERS ( string.h sys/time.h memory.h sys/types.h @@ -100,13 +100,13 @@ CHOOSE_FUNCTION(_realpath "realpath") CHOOSE_FUNCTION(_rename "rename") -CHOOSE_FUNCTION(_close "_close;close") -CHOOSE_FUNCTION(_read "_read;read") -CHOOSE_FUNCTION(_open "_open;open") -CHOOSE_FUNCTION(_write "_write;write") -CHOOSE_FUNCTION(_unlink "_unlink;unlink") +CHOOSE_FUNCTION(_close "_close(int);close") +CHOOSE_FUNCTION(_read "_read(int, void*, unsigned int);read") +CHOOSE_FUNCTION(_open "_open(const char*, int, ...);open") +CHOOSE_FUNCTION(_write "_write(int, const void*, unsigned int);write") +CHOOSE_FUNCTION(_unlink "_unlink(const char*);unlink") CHOOSE_FUNCTION(_ftime "_ftime;ftime") -CHOOSE_FUNCTION(_mkdir _mkdir) +CHOOSE_FUNCTION(_mkdir "_mkdir(const char*)") CHOOSE_FUNCTION(SLEEPFUNCTION "usleep;Sleep(0);_sleep") This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-07-21 17:11:00
|
Revision: 2790 http://clucene.svn.sourceforge.net/clucene/?rev=2790&view=rev Author: ustramooner Date: 2008-07-21 17:10:57 +0000 (Mon, 21 Jul 2008) Log Message: ----------- various solaris issues fixed. Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/search/DisjunctionSumScorer.cpp branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.cpp branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.h branches/lucene2_3_2/src/core/CLucene/util/inputstreambuffer.h branches/lucene2_3_2/src/test/testall.cpp Modified: branches/lucene2_3_2/src/core/CLucene/search/DisjunctionSumScorer.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/search/DisjunctionSumScorer.cpp 2008-07-21 16:48:54 UTC (rev 2789) +++ branches/lucene2_3_2/src/core/CLucene/search/DisjunctionSumScorer.cpp 2008-07-21 17:10:57 UTC (rev 2790) @@ -112,7 +112,7 @@ _CLTHROWA(CL_ERR_UnsupportedOperation,"UnsupportedOperationException: DisjunctionSumScorer::explain"); } -bool DisjunctionSumScorer::score( HitCollector* hc, int32_t max ) +bool DisjunctionSumScorer::score( HitCollector* hc, const int32_t max ) { while ( currentDoc < max ) { hc->collect( currentDoc, currentScore ); Modified: branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.cpp 2008-07-21 16:48:54 UTC (rev 2789) +++ branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.cpp 2008-07-21 17:10:57 UTC (rev 2790) @@ -27,6 +27,11 @@ #include "fileinputstream.h" #include <cerrno> #include <cstring> + +#ifdef _CL_HAVE_STRING_H + #include <string.h> +#endif + namespace jstreams { FileInputStream::FileInputStream(const char *filepath, int32_t buffersize) { Modified: branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.h 2008-07-21 16:48:54 UTC (rev 2789) +++ branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.h 2008-07-21 17:10:57 UTC (rev 2790) @@ -16,6 +16,7 @@ #ifndef FILEINPUTSTREAM_H #define FILEINPUTSTREAM_H +#include <stdio.h> #include "bufferedstream.h" namespace jstreams { Modified: branches/lucene2_3_2/src/core/CLucene/util/inputstreambuffer.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/inputstreambuffer.h 2008-07-21 16:48:54 UTC (rev 2789) +++ branches/lucene2_3_2/src/core/CLucene/util/inputstreambuffer.h 2008-07-21 17:10:57 UTC (rev 2790) @@ -26,6 +26,8 @@ #ifndef INPUTSTREAMBUFFER_H #define INPUTSTREAMBUFFER_H +#include <stdlib.h> + namespace jstreams { template <class T> Modified: branches/lucene2_3_2/src/test/testall.cpp =================================================================== --- branches/lucene2_3_2/src/test/testall.cpp 2008-07-21 16:48:54 UTC (rev 2789) +++ branches/lucene2_3_2/src/test/testall.cpp 2008-07-21 17:10:57 UTC (rev 2790) @@ -19,6 +19,9 @@ #ifdef _CL_HAVE_DIRECT_H #include <direct.h> #endif +#ifdef _CL_HAVE_UNISTD_H + #include <unistd.h> +#endif const char* cl_tempDir; bool cl_quiet; This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <syn...@us...> - 2008-07-22 18:48:58
|
Revision: 2795 http://clucene.svn.sourceforge.net/clucene/?rev=2795&view=rev Author: synhershko Date: 2008-07-22 18:48:54 +0000 (Tue, 22 Jul 2008) Log Message: ----------- Optimizing change in last commit Removing unnecessary class definition Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/index/_FieldsReader.h branches/lucene2_3_2/src/shared/CLucene/config/utf8.cpp Modified: branches/lucene2_3_2/src/core/CLucene/index/_FieldsReader.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/index/_FieldsReader.h 2008-07-22 18:37:51 UTC (rev 2794) +++ branches/lucene2_3_2/src/core/CLucene/index/_FieldsReader.h 2008-07-22 18:48:54 UTC (rev 2795) @@ -14,7 +14,7 @@ CL_CLASS_DEF(document,Field) CL_CLASS_DEF(index, FieldInfos) CL_CLASS_DEF(store,IndexInput) -CL_CLASS_DEF_JSTREAM(subinputstream) +//CL_CLASS_DEF_JSTREAM(subinputstream) CL_NS_DEF(index) Modified: branches/lucene2_3_2/src/shared/CLucene/config/utf8.cpp =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/config/utf8.cpp 2008-07-22 18:37:51 UTC (rev 2794) +++ branches/lucene2_3_2/src/shared/CLucene/config/utf8.cpp 2008-07-22 18:48:54 UTC (rev 2795) @@ -217,10 +217,10 @@ rp++; } - if ( static_cast<size_t>(sp-str) < result_length ) + size_t ret = sp-str; + if ( ret < result_length ) *rp = '\0'; - size_t ret = sp-str; return ret; } //get the number of bytes that make up the utf8 character. This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-07-23 09:58:55
|
Revision: 2798 http://clucene.svn.sourceforge.net/clucene/?rev=2798&view=rev Author: ustramooner Date: 2008-07-23 09:58:51 +0000 (Wed, 23 Jul 2008) Log Message: ----------- don't use shared clucene-shared library, use the files instead Modified Paths: -------------- branches/lucene2_3_2/src/core/CMakeLists.txt branches/lucene2_3_2/src/shared/CMakeLists.txt Modified: branches/lucene2_3_2/src/core/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/core/CMakeLists.txt 2008-07-23 09:58:08 UTC (rev 2797) +++ branches/lucene2_3_2/src/core/CMakeLists.txt 2008-07-23 09:58:51 UTC (rev 2798) @@ -125,12 +125,11 @@ INCLUDE_DIRECTORIES( ${clucene_SOURCE_DIR}/src/core ) add_library(clucene-core SHARED - ${clucene_core_Files} ${HEADERS} + ${clucene_core_Files} ${clucene_shared_Files} ${HEADERS} ) -TARGET_LINK_LIBRARIES(clucene-core clucene-shared-static) add_library(clucene-core-static STATIC - ${clucene_core_Files} ${HEADERS} + ${clucene_core_Files} ${clucene_shared_Files} ${HEADERS} ) #set properties on the libraries Modified: branches/lucene2_3_2/src/shared/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-07-23 09:58:08 UTC (rev 2797) +++ branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-07-23 09:58:51 UTC (rev 2798) @@ -245,33 +245,22 @@ SOURCE_GROUP("util" ./CLucene/util/*) SET(clucene_shared_Files - ./CLucene/SharedHeader.cpp - ./CLucene/config/gunichartables.cpp - ./CLucene/config/repl_tcslwr.cpp - ./CLucene/config/repl_tcstoll.cpp - ./CLucene/config/repl_tcscasecmp.cpp - ./CLucene/config/repl_tprintf.cpp - ./CLucene/config/repl_lltot.cpp - ./CLucene/config/repl_tcstod.cpp - ./CLucene/config/utf8.cpp - ./CLucene/config/threads.cpp - ./CLucene/debug/condition.cpp - ./CLucene/debug/error.cpp - ./CLucene/util/StringBuffer.cpp - ./CLucene/util/Misc.cpp - ./CLucene/util/dirent.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/SharedHeader.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/gunichartables.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tcslwr.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tcstoll.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tcscasecmp.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tprintf.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_lltot.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tcstod.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/utf8.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/threads.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/debug/condition.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/debug/error.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/util/StringBuffer.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/util/Misc.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp + + PARENT_SCOPE ) file(GLOB_RECURSE HEADERS ${clucene-shared_SOURCE_DIR}/*.h) - -add_library(clucene-shared-static STATIC - ${clucene_shared_Files} ${HEADERS} - #${clucene-shared_BINARY_DIR}/clucene-config.h ${clucene-shared_BINARY_DIR}/_clucene-config.h -) - -SET_TARGET_PROPERTIES(clucene-shared-static PROPERTIES - LINKER_LANGUAGE CXX - VERSION ${CPACK_PACKAGE_VERSION} - SOVERSION ${CPACK_PACKAGE_SOVERSION} - DEFINE_SYMBOL "" - COMPILE_DEFINITIONS_DEBUG _DEBUG -) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-07-24 10:03:24
|
Revision: 2803 http://clucene.svn.sourceforge.net/clucene/?rev=2803&view=rev Author: ustramooner Date: 2008-07-24 10:03:21 +0000 (Thu, 24 Jul 2008) Log Message: ----------- check stdcall function in direct.h also... fixed test compilation shared files Modified Paths: -------------- branches/lucene2_3_2/src/shared/cmake/CheckStdCallFunctionExists.cmake branches/lucene2_3_2/src/test/CMakeLists.txt Modified: branches/lucene2_3_2/src/shared/cmake/CheckStdCallFunctionExists.cmake =================================================================== --- branches/lucene2_3_2/src/shared/cmake/CheckStdCallFunctionExists.cmake 2008-07-24 09:12:08 UTC (rev 2802) +++ branches/lucene2_3_2/src/shared/cmake/CheckStdCallFunctionExists.cmake 2008-07-24 10:03:21 UTC (rev 2803) @@ -29,6 +29,9 @@ IF ( HAVE_UNISTD_H ) SET(CHECK_STDCALL_FUNCTION_PREMAIN "${CHECK_STDCALL_FUNCTION_PREMAIN}#include \"unistd.h\"\n") ENDIF ( HAVE_UNISTD_H ) + IF ( HAVE_DIRECT_H ) + SET(CHECK_STDCALL_FUNCTION_PREMAIN "${CHECK_STDCALL_FUNCTION_PREMAIN}#include \"direct.h\"\n") + ENDIF ( HAVE_DIRECT_H ) STRING(REGEX REPLACE "(\\(.*\\))" "" CHECK_STDCALL_FUNCTION_EXISTS_FUNCTION ${FUNCTION_DECLARATION} ) Modified: branches/lucene2_3_2/src/test/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/test/CMakeLists.txt 2008-07-24 09:12:08 UTC (rev 2802) +++ branches/lucene2_3_2/src/test/CMakeLists.txt 2008-07-24 10:03:21 UTC (rev 2803) @@ -42,9 +42,9 @@ ./util/English.cpp ${test_HEADERS} ) - + #todo: do glob header and include header files for IDE. -ADD_EXECUTABLE(cl_test ${test_files} ) +ADD_EXECUTABLE(cl_test ${clucene_shared_Files} ${test_files} ) SET_TARGET_PROPERTIES(cl_test PROPERTIES DEFINE_SYMBOL "") IF (TESTS_USE_SHARED_LIBRARY) @@ -62,8 +62,10 @@ SET(test_monolithic_Files ./CLMonolithic_Test.cpp ${clucene-core_SOURCE_DIR}/CLucene/CLMonolithic.cpp) SET(CMAKE_CXX_FLAGS_DEBUG) -CHECK_CXX_ACCEPTS_FLAG(-Wall GccFlagWall) -CHECK_CXX_ACCEPTS_FLAG(-pedantic GccFlagPedantic) +IF ( CMAKE_COMPILER_IS_GNUCC ) + CHECK_CXX_ACCEPTS_FLAG(-Wall GccFlagWall) + CHECK_CXX_ACCEPTS_FLAG(-pedantic GccFlagPedantic) +ENDIF ( CMAKE_COMPILER_IS_GNUCC ) IF ( GccFlagG ) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") @@ -136,6 +138,7 @@ COMMAND ${CMAKE_COMMAND} test-mmap COMMAND ${CMAKE_COMMAND} test-singlethreading COMMAND ${CMAKE_COMMAND} test-refcnt + DEPENDS cl_test-pedantic cl_test-ascii cl_test-namespace cl_test-mmap cl_test-singlethreading cl_test-refcnt cl_test-platform-charfuncs ) ENDIF ( ENABLE_COMPILE_TESTS ) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-07-24 10:37:56
|
Revision: 2805 http://clucene.svn.sourceforge.net/clucene/?rev=2805&view=rev Author: ustramooner Date: 2008-07-24 10:37:52 +0000 (Thu, 24 Jul 2008) Log Message: ----------- backport cmake 2.4 (problems with scope, i think) Modified Paths: -------------- branches/lucene2_3_2/src/core/CMakeLists.txt branches/lucene2_3_2/src/shared/CMakeLists.txt branches/lucene2_3_2/src/test/CMakeLists.txt Modified: branches/lucene2_3_2/src/core/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/core/CMakeLists.txt 2008-07-24 10:15:34 UTC (rev 2804) +++ branches/lucene2_3_2/src/core/CMakeLists.txt 2008-07-24 10:37:52 UTC (rev 2805) @@ -117,6 +117,8 @@ ./CLucene/search/DateFilter.cpp ) +GET_SHARED_FILES(clucene_core_Files) + #find our headers file(GLOB_RECURSE HEADERS ${clucene-core_SOURCE_DIR}/*.h) Modified: branches/lucene2_3_2/src/shared/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-07-24 10:15:34 UTC (rev 2804) +++ branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-07-24 10:37:52 UTC (rev 2805) @@ -219,23 +219,25 @@ SOURCE_GROUP("debug" ./CLucene/debug/*) SOURCE_GROUP("util" ./CLucene/util/*) -SET(clucene_shared_Files - ${clucene-shared_SOURCE_DIR}/CLucene/SharedHeader.cpp - ${clucene-shared_SOURCE_DIR}/CLucene/config/gunichartables.cpp - ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tcslwr.cpp - ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tcstoll.cpp - ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tcscasecmp.cpp - ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tprintf.cpp - ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_lltot.cpp - ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tcstod.cpp - ${clucene-shared_SOURCE_DIR}/CLucene/config/utf8.cpp - ${clucene-shared_SOURCE_DIR}/CLucene/config/threads.cpp - ${clucene-shared_SOURCE_DIR}/CLucene/debug/condition.cpp - ${clucene-shared_SOURCE_DIR}/CLucene/debug/error.cpp - ${clucene-shared_SOURCE_DIR}/CLucene/util/StringBuffer.cpp - ${clucene-shared_SOURCE_DIR}/CLucene/util/Misc.cpp - ${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp - - PARENT_SCOPE +MACRO (GET_SHARED_FILES result) + SET(${result} + ${clucene-shared_SOURCE_DIR}/CLucene/SharedHeader.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/gunichartables.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tcslwr.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tcstoll.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tcscasecmp.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tprintf.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_lltot.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/repl_tcstod.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/utf8.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/config/threads.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/debug/condition.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/debug/error.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/util/StringBuffer.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/util/Misc.cpp + ${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp ) + +ENDMACRO (GET_SHARED_FILES) + file(GLOB_RECURSE HEADERS ${clucene-shared_SOURCE_DIR}/*.h) Modified: branches/lucene2_3_2/src/test/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/test/CMakeLists.txt 2008-07-24 10:15:34 UTC (rev 2804) +++ branches/lucene2_3_2/src/test/CMakeLists.txt 2008-07-24 10:37:52 UTC (rev 2805) @@ -42,6 +42,7 @@ ./util/English.cpp ${test_HEADERS} ) +GET_SHARED_FILES(clucene_core_Files) #todo: do glob header and include header files for IDE. ADD_EXECUTABLE(cl_test ${clucene_shared_Files} ${test_files} ) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-07-24 10:57:42
|
Revision: 2806 http://clucene.svn.sourceforge.net/clucene/?rev=2806&view=rev Author: ustramooner Date: 2008-07-24 10:57:39 +0000 (Thu, 24 Jul 2008) Log Message: ----------- Fixed clucene_shared_files Modified Paths: -------------- branches/lucene2_3_2/src/core/CMakeLists.txt branches/lucene2_3_2/src/shared/CMakeLists.txt branches/lucene2_3_2/src/test/CMakeLists.txt Modified: branches/lucene2_3_2/src/core/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/core/CMakeLists.txt 2008-07-24 10:37:52 UTC (rev 2805) +++ branches/lucene2_3_2/src/core/CMakeLists.txt 2008-07-24 10:57:39 UTC (rev 2806) @@ -117,7 +117,7 @@ ./CLucene/search/DateFilter.cpp ) -GET_SHARED_FILES(clucene_core_Files) +GET_SHARED_FILES(clucene_shared_Files) #find our headers file(GLOB_RECURSE HEADERS ${clucene-core_SOURCE_DIR}/*.h) Modified: branches/lucene2_3_2/src/shared/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-07-24 10:37:52 UTC (rev 2805) +++ branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-07-24 10:57:39 UTC (rev 2806) @@ -236,8 +236,7 @@ ${clucene-shared_SOURCE_DIR}/CLucene/util/StringBuffer.cpp ${clucene-shared_SOURCE_DIR}/CLucene/util/Misc.cpp ${clucene-shared_SOURCE_DIR}/CLucene/util/dirent.cpp -) - + ) ENDMACRO (GET_SHARED_FILES) file(GLOB_RECURSE HEADERS ${clucene-shared_SOURCE_DIR}/*.h) Modified: branches/lucene2_3_2/src/test/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/test/CMakeLists.txt 2008-07-24 10:37:52 UTC (rev 2805) +++ branches/lucene2_3_2/src/test/CMakeLists.txt 2008-07-24 10:57:39 UTC (rev 2806) @@ -42,7 +42,7 @@ ./util/English.cpp ${test_HEADERS} ) -GET_SHARED_FILES(clucene_core_Files) +GET_SHARED_FILES(clucene_shared_Files) #todo: do glob header and include header files for IDE. ADD_EXECUTABLE(cl_test ${clucene_shared_Files} ${test_files} ) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <syn...@us...> - 2008-07-24 13:12:19
|
Revision: 2809 http://clucene.svn.sourceforge.net/clucene/?rev=2809&view=rev Author: synhershko Date: 2008-07-24 13:12:14 +0000 (Thu, 24 Jul 2008) Log Message: ----------- Deprecating termText to conform with JLucene 2.3.2 Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.cpp branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.h branches/lucene2_3_2/src/core/CLucene/analysis/Analyzers.cpp branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardFilter.cpp branches/lucene2_3_2/src/core/CLucene/document/DateTools.cpp branches/lucene2_3_2/src/core/CLucene/index/DocumentWriter.cpp branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParserBase.cpp branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h branches/lucene2_3_2/src/test/analysis/TestAnalysis.cpp branches/lucene2_3_2/src/test/analysis/TestAnalyzers.cpp branches/lucene2_3_2/src/test/queryParser/TestQueryParser.cpp Modified: branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.cpp 2008-07-24 11:02:57 UTC (rev 2808) +++ branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.cpp 2008-07-24 13:12:14 UTC (rev 2809) @@ -153,6 +153,11 @@ _termTextLen = _tcslen(_termText); return _termTextLen; } +size_t Token::termLength() { + if ( _termTextLen == -1 ) //it was invalidated by growBuffer + _termTextLen = _tcslen(_termText); + return _termTextLen; +} void Token::resetTermTextLen(){ _termTextLen=-1; } Modified: branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.h 2008-07-24 11:02:57 UTC (rev 2808) +++ branches/lucene2_3_2/src/core/CLucene/analysis/AnalysisHeader.h 2008-07-24 13:12:14 UTC (rev 2809) @@ -108,11 +108,6 @@ void setPositionIncrement(int32_t posIncr); int32_t getPositionIncrement() const; - const TCHAR* termText() const; //< See #termBuffer() - size_t termTextLength(); //< Length of the the termBuffer. See #termBuffer - void resetTermTextLen(); //< Empties the termBuffer. See #termBuffer - void setText(const TCHAR* txt); //< Sets the termBuffer. See #termBuffer - /** Returns the internal termBuffer character array which * you can then directly alter. If the array is too * small for your token, use {@link @@ -121,7 +116,14 @@ * #setTermLength} to record the number of valid * characters that were placed into the termBuffer. */ const TCHAR* termBuffer() const; + size_t termLength(); //< Length of the the termBuffer. See #termBuffer + _CL_DEPRECATED( termBuffer ) const TCHAR* termText() const; //< See #termBuffer() + _CL_DEPRECATED( termLength ) size_t termTextLength(); //< See #termLength + + void resetTermTextLen(); //< Empties the termBuffer. See #termBuffer + void setText(const TCHAR* txt); //< Sets the termBuffer. See #termBuffer + /** * Returns this Token's starting offset, the position of the first character * corresponding to this token in the source text. Modified: branches/lucene2_3_2/src/core/CLucene/analysis/Analyzers.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/analysis/Analyzers.cpp 2008-07-24 11:02:57 UTC (rev 2808) +++ branches/lucene2_3_2/src/core/CLucene/analysis/Analyzers.cpp 2008-07-24 13:12:14 UTC (rev 2809) @@ -313,8 +313,8 @@ } bool ISOLatin1AccentFilter::next(Token* token){ if ( input->next(token) ){ - int32_t l = token->termTextLength(); - const TCHAR* chars = token->termText(); + int32_t l = token->termLength(); + const TCHAR* chars = token->termBuffer(); bool doProcess = false; for (int32_t i = 0; i < l; ++i) { #ifdef _UCS2 @@ -538,7 +538,7 @@ // return the first non-stop word found while ( input->next(token) ) { - size_t len = token->termTextLength(); + size_t len = token->termLength(); if (len >= _min && len <= _max) return true; // note: else we ignore it but should we index each part of it? Modified: branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardFilter.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardFilter.cpp 2008-07-24 11:02:57 UTC (rev 2808) +++ branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardFilter.cpp 2008-07-24 13:12:14 UTC (rev 2809) @@ -29,7 +29,7 @@ return false; TCHAR* text = t->_termText; - const int32_t textLength = t->termTextLength(); + const int32_t textLength = t->termLength(); const TCHAR* type = t->type(); if ( type == tokenImage[APOSTROPHE] && //we can compare the type directy since the type should always come from the tokenImage Modified: branches/lucene2_3_2/src/core/CLucene/document/DateTools.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/document/DateTools.cpp 2008-07-24 11:02:57 UTC (rev 2808) +++ branches/lucene2_3_2/src/core/CLucene/document/DateTools.cpp 2008-07-24 13:12:14 UTC (rev 2809) @@ -48,7 +48,7 @@ if (resolution == MILLISECOND_FORMAT) { size_t len = strftime(abuf, DATETOOLS_BUFFER_SIZE, "%Y%m%d%H%M%S", ptm); - uint32_t ms = time % 1000; + uint32_t ms = static_cast<uint32_t>(time % 1000); _snprintf(abuf + len, 4, "%03u", ms); } else if (resolution == SECOND_FORMAT) { strftime(abuf, DATETOOLS_BUFFER_SIZE, "%Y%m%d%H%M%S", ptm); Modified: branches/lucene2_3_2/src/core/CLucene/index/DocumentWriter.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/index/DocumentWriter.cpp 2008-07-24 11:02:57 UTC (rev 2808) +++ branches/lucene2_3_2/src/core/CLucene/index/DocumentWriter.cpp 2008-07-24 13:12:14 UTC (rev 2809) @@ -300,9 +300,9 @@ TermVectorOffsetInfo tio; tio.setStartOffset(offset + t.startOffset()); tio.setEndOffset(offset + t.endOffset()); - addPosition(fieldName, t.termText(), position++, &tio); + addPosition(fieldName, t.termBuffer(), position++, &tio); }else - addPosition(fieldName, t.termText(), position++, NULL); + addPosition(fieldName, t.termBuffer(), position++, NULL); lastTokenEndOffset = t.endOffset(); length++; Modified: branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParserBase.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParserBase.cpp 2008-07-24 11:02:57 UTC (rev 2808) +++ branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParserBase.cpp 2008-07-24 13:12:14 UTC (rev 2809) @@ -155,7 +155,7 @@ //Get the tokens from the source try{ while (source->next(&t)){ - v.push_back(STRDUP_TtoT(t.termText())); + v.push_back(STRDUP_TtoT(t.termBuffer())); if (t.getPositionIncrement() != 0) positionCount += t.getPositionIncrement(); @@ -251,11 +251,11 @@ } if (tret) { - if ( !from && _tcscmp(t.termText(),_T("TO"))==0 ) + if ( !from && _tcscmp(t.termBuffer(),_T("TO"))==0 ) continue; - TCHAR* tmp = STRDUP_TtoT(t.termText()); + TCHAR* tmp = STRDUP_TtoT(t.termBuffer()); discardEscapeChar(tmp); terms[from? 0 : 1] = tmp; Modified: branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-07-24 11:02:57 UTC (rev 2808) +++ branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-07-24 13:12:14 UTC (rev 2809) @@ -101,7 +101,6 @@ #define CL_NS2(sub,sub2) #define CL_CLASS_DEF(sub,clazz) class clazz; #endif -#define CL_CLASS_DEF_JSTREAM(clazz) namespace jstream{ class clazz; } #if defined(LUCENE_NO_STDC_NAMESPACE) //todo: haven't actually tested this on a non-stdc compliant compiler Modified: branches/lucene2_3_2/src/test/analysis/TestAnalysis.cpp =================================================================== --- branches/lucene2_3_2/src/test/analysis/TestAnalysis.cpp 2008-07-24 11:02:57 UTC (rev 2808) +++ branches/lucene2_3_2/src/test/analysis/TestAnalysis.cpp 2008-07-24 13:12:14 UTC (rev 2809) @@ -16,7 +16,7 @@ Token t; for (; stream->next(&t);) { if (verbose) { - CuMessage(tc, _T("Text=%s start=%d end=%d\n"), t.termText(), t.startOffset(), t.endOffset() ); + CuMessage(tc, _T("Text=%s start=%d end=%d\n"), t.termBuffer(), t.startOffset(), t.endOffset() ); } // _CLDELETE(t); count++; Modified: branches/lucene2_3_2/src/test/analysis/TestAnalyzers.cpp =================================================================== --- branches/lucene2_3_2/src/test/analysis/TestAnalyzers.cpp 2008-07-24 11:02:57 UTC (rev 2808) +++ branches/lucene2_3_2/src/test/analysis/TestAnalyzers.cpp 2008-07-24 13:12:14 UTC (rev 2809) @@ -21,7 +21,7 @@ buffer[len]=0; CLUCENE_ASSERT(ts->next(&t)); - CLUCENE_ASSERT(_tcscmp( t.termText(),buffer) == 0 ); + CLUCENE_ASSERT(_tcscmp( t.termBuffer(),buffer) == 0 ); last = pos+1; } @@ -67,15 +67,15 @@ Token token; CLUCENE_ASSERT( tokenStream->next(&token) ); - CuAssertStrEquals(tc,_T("token.termText()"), _T("Qwerty"), - token.termText()); + CuAssertStrEquals(tc,_T("token.termBuffer()"), _T("Qwerty"), + token.termBuffer()); _CLDELETE(tokenStream); StringReader reader2(text); tokenStream = analyzer.tokenStream(_T("special"), &reader2); CLUCENE_ASSERT( tokenStream->next(&token) ); - CuAssertStrEquals(tc, _T("token.termText()"), _T("qwerty"), - token.termText()); + CuAssertStrEquals(tc, _T("token.termBuffer()"), _T("qwerty"), + token.termBuffer()); _CLDELETE(tokenStream); } @@ -123,79 +123,79 @@ Token token; - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("Des"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("mot"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("cles"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("A"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("LA"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("CHAINE"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("A"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("A"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("A"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("A"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("A"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("A"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("AE"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("C"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("E"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("E"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("E"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("E"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("I"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("I"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("I"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("I"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("D"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("N"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("O"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("O"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("O"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("O"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("O"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("O"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("TH"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("U"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("U"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("U"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("U"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("Y"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("a"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("a"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("a"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("a"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("a"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("a"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("ae"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("c"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("e"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("e"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("e"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("e"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("i"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("i"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("i"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("i"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("d"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("n"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("o"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("o"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("o"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("o"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("o"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("o"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("ss"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("th"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("u"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("u"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("u"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("u"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("y"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("y"), token.termText()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("Des"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("mot"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("cles"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("A"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("LA"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("CHAINE"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("A"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("A"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("A"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("A"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("A"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("A"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("AE"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("C"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("E"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("E"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("E"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("E"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("I"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("I"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("I"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("I"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("D"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("N"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("O"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("O"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("O"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("O"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("O"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("O"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("TH"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("U"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("U"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("U"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("U"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("Y"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("a"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("a"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("a"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("a"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("a"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("a"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("ae"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("c"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("e"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("e"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("e"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("e"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("i"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("i"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("i"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("i"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("d"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("n"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("o"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("o"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("o"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("o"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("o"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("o"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("ss"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("th"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("u"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("u"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("u"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("u"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("y"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("y"), token.termBuffer()); #ifdef _UCS2 - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("OE"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("oe"), token.termText()); - CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("Y"), token.termText()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("OE"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("oe"), token.termBuffer()); + CLUCENE_ASSERT(filter.next(&token)); CuAssertStrEquals(tc, _T("Token compare"), _T("Y"), token.termBuffer()); #endif Modified: branches/lucene2_3_2/src/test/queryParser/TestQueryParser.cpp =================================================================== --- branches/lucene2_3_2/src/test/queryParser/TestQueryParser.cpp 2008-07-24 11:02:57 UTC (rev 2808) +++ branches/lucene2_3_2/src/test/queryParser/TestQueryParser.cpp 2008-07-24 13:12:14 UTC (rev 2809) @@ -31,13 +31,13 @@ return true; }else{ while( input->next(token) ){ - if ( _tcscmp(token->termText(), _T("phrase")) == 0 ) { + if ( _tcscmp(token->termBuffer(), _T("phrase")) == 0 ) { inPhrase = true; savedStart = token->startOffset(); savedEnd = token->endOffset(); token->set( _T("phrase1"), savedStart, savedEnd); return true; - }else if ( _tcscmp(token->termText(), _T("stop") ) !=0 ){ + }else if ( _tcscmp(token->termBuffer(), _T("stop") ) !=0 ){ return true; } } This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-07-24 15:04:18
|
Revision: 2811 http://clucene.svn.sourceforge.net/clucene/?rev=2811&view=rev Author: ustramooner Date: 2008-07-24 15:04:11 +0000 (Thu, 24 Jul 2008) Log Message: ----------- get installation targets working again. few small changes made for strigi compatibility Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene.h branches/lucene2_3_2/src/core/CMakeLists.txt branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h branches/lucene2_3_2/src/shared/CLucene/debug/_condition.h branches/lucene2_3_2/src/shared/CLucene/debug/mem.h branches/lucene2_3_2/src/shared/CMakeLists.txt branches/lucene2_3_2/src/test/CMakeLists.txt Modified: branches/lucene2_3_2/src/core/CLucene.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene.h 2008-07-24 13:21:19 UTC (rev 2810) +++ branches/lucene2_3_2/src/core/CLucene.h 2008-07-24 15:04:11 UTC (rev 2811) @@ -23,6 +23,7 @@ #include "CLucene/search/RangeQuery.h" #include "CLucene/search/BooleanQuery.h" #include "CLucene/search/TermQuery.h" +#include "CLucene/search/SearchHeader.h" #include "CLucene/search/Similarity.h" #include "CLucene/search/Sort.h" #include "CLucene/search/Hits.h" Modified: branches/lucene2_3_2/src/core/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/core/CMakeLists.txt 2008-07-24 13:21:19 UTC (rev 2810) +++ branches/lucene2_3_2/src/core/CMakeLists.txt 2008-07-24 15:04:11 UTC (rev 2811) @@ -154,12 +154,13 @@ get_filename_component(apath ${file} PATH) get_filename_component(aname ${file} NAME) file(RELATIVE_PATH relpath ${CMAKE_SOURCE_DIR}/src/core ${apath}) + IF ( NOT aname MATCHES "^_.*" ) + install(FILES ${file} DESTINATION include/${relpath}) + ENDIF ( NOT aname MATCHES "^_.*" ) ENDFOREACH(file) #also install CLucene.h -install(FILES ${clucene-shared_BINARY_DIR}/clucene-config.h DESTINATION lib/CLucene ) +install(FILES ${clucene-shared_BINARY_DIR}/CLucene/clucene-config.h DESTINATION lib/CLucene ) -install(FILES ${HEADERS} DESTINATION include/) - #and install libraries install(TARGETS clucene-core DESTINATION lib) install(TARGETS clucene-core-static DESTINATION lib) Modified: branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-07-24 13:21:19 UTC (rev 2810) +++ branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-07-24 15:04:11 UTC (rev 2811) @@ -140,13 +140,15 @@ #define _CL_CONST_CAST(typ,var) const_cast<typ>(var) #define _CL_REINTERPRET_CAST(typ,var) reinterpret_cast<typ>(var) -//todo: put this logic in cmake -#if _MSC_FULL_VER >= 140050320 - #define _CL_DEPRECATE_TEXT(_Text) __declspec(deprecated(_Text)) -#elif _MSC_VER >= 1300 - #define _CL_DEPRECATE_TEXT(_Text) __declspec(deprecated) -#elif defined(_MSC_VER) - #define _CL_DEPRECATE_TEXT(_Text) +//todo: put this logic in cmake +#if defined(_MSC_VER) + #if _MSC_FULL_VER >= 140050320 + #define _CL_DEPRECATE_TEXT(_Text) __declspec(deprecated(_Text)) + #elif _MSC_VER >= 1300 + #define _CL_DEPRECATE_TEXT(_Text) __declspec(deprecated) + #else + #define _CL_DEPRECATE_TEXT(_Text) + #endif #elif (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1)) #define _CL_DEPRECATE_TEXT(_Text) __attribute__((__deprecated__)) #else Modified: branches/lucene2_3_2/src/shared/CLucene/debug/_condition.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/debug/_condition.h 2008-07-24 13:21:19 UTC (rev 2810) +++ branches/lucene2_3_2/src/shared/CLucene/debug/_condition.h 2008-07-24 15:04:11 UTC (rev 2811) @@ -7,6 +7,8 @@ #ifndef __CONDITION_H #define __CONDITION_H +//todo: this is a hack +#undef CND_PRECONDITION /* To enable condition debugging uncomment _CND_DEBUG in CLConfig.h @@ -51,7 +53,7 @@ #define CND__CONDITION(cond,file,line,title,mes2) {if(!(cond)){__cnd_FormatDebug(file,line,title,mes2,0);}} #define CND__MESSAGE(file,line,title,mes2) {__cnd_FormatDebug(file,line,title,mes2,0);} #else - #define CND_PRECONDITION(cond, usermessage) + #define CND_PRECONDITION(cond, usermessage) #define CND_CONDITION(cond, usermessage) #define CND_WARNING(cond,usermessage) #define CND_MESSAGE(cond,usermessage) Modified: branches/lucene2_3_2/src/shared/CLucene/debug/mem.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/debug/mem.h 2008-07-24 13:21:19 UTC (rev 2810) +++ branches/lucene2_3_2/src/shared/CLucene/debug/mem.h 2008-07-24 15:04:11 UTC (rev 2811) @@ -7,6 +7,10 @@ #ifndef _lucene_debug_mem_h #define _lucene_debug_mem_h +//todo: this is a hack... +#ifndef CND_PRECONDITION + #define CND_PRECONDITION(x,y) +#endif //Macro for creating new objects #if defined(LUCENE_ENABLE_REFCOUNT) @@ -27,7 +31,7 @@ #define LUCENE_BASE_CHECK(x) #endif -#if (_MSC_VER < 1300) +#if defined(_MSC_VER) && (_MSC_VER < 1300) //6.0 #define _CLDELETE_CARRAY(x) if (x!=NULL){delete[] _CL_CONST_CAST(TCHAR*,x); x=NULL;} #define _CLDELETE_CaARRAY(x) if (x!=NULL){delete[] _CL_CONST_CAST(char*,x); x=NULL;} Modified: branches/lucene2_3_2/src/shared/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-07-24 13:21:19 UTC (rev 2810) +++ branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-07-24 15:04:11 UTC (rev 2811) @@ -239,4 +239,10 @@ ) ENDMACRO (GET_SHARED_FILES) -file(GLOB_RECURSE HEADERS ${clucene-shared_SOURCE_DIR}/*.h) +#install public headers. +install(FILES ${clucene-shared_SOURCE_DIR}/CLucene/SharedHeader.h DESTINATION include/CLucene ) +install(FILES ${clucene-shared_SOURCE_DIR}/CLucene/LuceneThreads.h DESTINATION include/CLucene ) +install(FILES ${clucene-shared_SOURCE_DIR}/CLucene/debug/lucenebase.h DESTINATION include/CLucene/debug ) +install(FILES ${clucene-shared_SOURCE_DIR}/CLucene/debug/mem.h DESTINATION include/CLucene/debug ) +install(FILES ${clucene-shared_SOURCE_DIR}/CLucene/debug/error.h DESTINATION include/CLucene/debug ) + Modified: branches/lucene2_3_2/src/test/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/test/CMakeLists.txt 2008-07-24 13:21:19 UTC (rev 2810) +++ branches/lucene2_3_2/src/test/CMakeLists.txt 2008-07-24 15:04:11 UTC (rev 2811) @@ -45,7 +45,7 @@ GET_SHARED_FILES(clucene_shared_Files) #todo: do glob header and include header files for IDE. -ADD_EXECUTABLE(cl_test ${clucene_shared_Files} ${test_files} ) +ADD_EXECUTABLE(cl_test EXCLUDE_FROM_ALL ${clucene_shared_Files} ${test_files} ) SET_TARGET_PROPERTIES(cl_test PROPERTIES DEFINE_SYMBOL "") IF (TESTS_USE_SHARED_LIBRARY) @@ -78,7 +78,7 @@ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic") ENDIF ( GccFlagPedantic ) -ADD_EXECUTABLE(cl_test-pedantic ${test_monolithic_Files}) +ADD_EXECUTABLE(cl_test-pedantic EXCLUDE_FROM_ALL ${test_monolithic_Files}) SET_TARGET_PROPERTIES(cl_test-pedantic PROPERTIES COMPILE_DEFINITIONS "") ADD_CUSTOM_TARGET(test-pedantic COMMENT "Running cl_test-pedantic" This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <syn...@us...> - 2008-07-24 16:52:01
|
Revision: 2812 http://clucene.svn.sourceforge.net/clucene/?rev=2812&view=rev Author: synhershko Date: 2008-07-24 16:51:58 +0000 (Thu, 24 Jul 2008) Log Message: ----------- Removing unnecessary macros Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/search/FieldDocSortedHitQueue.cpp branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h Modified: branches/lucene2_3_2/src/core/CLucene/search/FieldDocSortedHitQueue.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/search/FieldDocSortedHitQueue.cpp 2008-07-24 15:04:11 UTC (rev 2811) +++ branches/lucene2_3_2/src/core/CLucene/search/FieldDocSortedHitQueue.cpp 2008-07-24 16:51:58 UTC (rev 2812) @@ -44,7 +44,7 @@ } bool FieldDocSortedHitQueue::lessThan (FieldDoc* docA, FieldDoc* docB) { - int32_t n = fieldsLen; + const int32_t n = fieldsLen; int32_t c = 0; float_t f1,f2,r1,r2; int32_t i1,i2; @@ -55,21 +55,21 @@ if (fields[i]->getReverse()) { switch (type) { case SortField::DOCSCORE: - r1 = _CL_REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); - r2 = _CL_REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); + r1 = reinterpret_cast<Compare::Float*>(docA->fields[i])->getValue(); + r2 = reinterpret_cast<Compare::Float*>(docB->fields[i])->getValue(); if (r1 < r2) c = -1; if (r1 > r2) c = 1; break; case SortField::DOC: case SortField::INT: - i1 = _CL_REINTERPRET_CAST(Compare::Int32*, docA->fields[i])->getValue(); - i2 = _CL_REINTERPRET_CAST(Compare::Int32*, docB->fields[i])->getValue(); + i1 = reinterpret_cast<Compare::Int32*>(docA->fields[i])->getValue(); + i2 = reinterpret_cast<Compare::Int32*>(docB->fields[i])->getValue(); if (i1 > i2) c = -1; if (i1 < i2) c = 1; break; case SortField::STRING: - s1 = _CL_REINTERPRET_CAST(Compare::TChar*, docA->fields[i])->getValue(); - s2 = _CL_REINTERPRET_CAST(Compare::TChar*, docB->fields[i])->getValue(); + s1 = reinterpret_cast<Compare::TChar*>(docA->fields[i])->getValue(); + s2 = reinterpret_cast<Compare::TChar*>(docB->fields[i])->getValue(); if (s2 == NULL) c = -1; // could be NULL if there are else if (s1 == NULL) c = 1; // no terms in the given field else c = _tcscmp(s2,s1); //else if (fields[i].getLocale() == NULL) { @@ -80,8 +80,8 @@ }*/ break; case SortField::FLOAT: - f1 = _CL_REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); - f2 = _CL_REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); + f1 = reinterpret_cast<Compare::Float*>(docA->fields[i])->getValue(); + f2 = reinterpret_cast<Compare::Float*>(docB->fields[i])->getValue(); if (f1 > f2) c = -1; if (f1 < f2) c = 1; break; @@ -100,21 +100,21 @@ } else { switch (type) { case SortField::DOCSCORE: - r1 = _CL_REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); - r2 = _CL_REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); + r1 = reinterpret_cast<Compare::Float*>(docA->fields[i])->getValue(); + r2 = reinterpret_cast<Compare::Float*>(docB->fields[i])->getValue(); if (r1 > r2) c = -1; if (r1 < r2) c = 1; break; case SortField::DOC: case SortField::INT: - i1 = _CL_REINTERPRET_CAST(Compare::Int32*, docA->fields[i])->getValue(); - i2 = _CL_REINTERPRET_CAST(Compare::Int32*, docB->fields[i])->getValue(); + i1 = reinterpret_cast<Compare::Int32*>(docA->fields[i])->getValue(); + i2 = reinterpret_cast<Compare::Int32*>(docB->fields[i])->getValue(); if (i1 < i2) c = -1; if (i1 > i2) c = 1; break; case SortField::STRING: - s1 = _CL_REINTERPRET_CAST(Compare::TChar*, docA->fields[i])->getValue(); - s2 = _CL_REINTERPRET_CAST(Compare::TChar*, docB->fields[i])->getValue(); + s1 = reinterpret_cast<Compare::TChar*>(docA->fields[i])->getValue(); + s2 = reinterpret_cast<Compare::TChar*>(docB->fields[i])->getValue(); // NULL values need to be sorted first, because of how FieldCache.getStringIndex() // works - in that routine, any documents without a value in the given field are // put first. @@ -127,8 +127,8 @@ }*/ break; case SortField::FLOAT: - f1 = _CL_REINTERPRET_CAST(Compare::Float*, docA->fields[i])->getValue(); - f2 = _CL_REINTERPRET_CAST(Compare::Float*, docB->fields[i])->getValue(); + f1 = reinterpret_cast<Compare::Float*>(docA->fields[i])->getValue(); + f2 = reinterpret_cast<Compare::Float*>(docB->fields[i])->getValue(); if (f1 < f2) c = -1; if (f1 > f2) c = 1; break; Modified: branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-07-24 15:04:11 UTC (rev 2811) +++ branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-07-24 16:51:58 UTC (rev 2812) @@ -137,9 +137,6 @@ //////////////////////////////////////////////////////// -#define _CL_CONST_CAST(typ,var) const_cast<typ>(var) -#define _CL_REINTERPRET_CAST(typ,var) reinterpret_cast<typ>(var) - //todo: put this logic in cmake #if defined(_MSC_VER) #if _MSC_FULL_VER >= 140050320 This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-07-31 13:02:28
|
Revision: 2837 http://clucene.svn.sourceforge.net/clucene/?rev=2837&view=rev Author: ustramooner Date: 2008-07-31 13:02:19 +0000 (Thu, 31 Jul 2008) Log Message: ----------- reworked the definitions for exports Modified Paths: -------------- branches/lucene2_3_2/src/contribs/CMakeLists.txt branches/lucene2_3_2/src/core/CMakeLists.txt branches/lucene2_3_2/src/demo/CMakeLists.txt branches/lucene2_3_2/src/shared/CLucene/LuceneThreads.h branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h branches/lucene2_3_2/src/shared/CLucene/_SharedHeader.h branches/lucene2_3_2/src/shared/CLucene/debug/error.h branches/lucene2_3_2/src/shared/CMakeLists.txt Modified: branches/lucene2_3_2/src/contribs/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/contribs/CMakeLists.txt 2008-07-30 18:23:09 UTC (rev 2836) +++ branches/lucene2_3_2/src/contribs/CMakeLists.txt 2008-07-31 13:02:19 UTC (rev 2837) @@ -2,6 +2,7 @@ INCLUDE (DefineOptions) DEFINE_OPTIONS(EXTRA_OPTIONS) +ADD_DEFINITIONS(${EXTRA_OPTIONS} -DMAKE_CLUCENE_CONTRIBS_LIB) set(CMAKE_MODULE_PATH "${clucene-contribs_SOURCE_DIR}/cmake") @@ -152,13 +153,11 @@ SET_TARGET_PROPERTIES(clucene-contribs PROPERTIES VERSION ${CLUCENE_VERSION} SOVERSION ${CLUCENE_SOVERSION} - DEFINE_SYMBOL "MAKE_CLUCENE_CONTRIBS_LIB" COMPILE_DEFINITIONS_DEBUG _DEBUG ) SET_TARGET_PROPERTIES(clucene-contribs-static PROPERTIES VERSION ${CLUCENE_VERSION} SOVERSION ${CLUCENE_SOVERSION} - DEFINE_SYMBOL "MAKE_CLUCENE_CONTRIBS_LIB" COMPILE_DEFINITIONS_DEBUG _DEBUG ) Modified: branches/lucene2_3_2/src/core/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/core/CMakeLists.txt 2008-07-30 18:23:09 UTC (rev 2836) +++ branches/lucene2_3_2/src/core/CMakeLists.txt 2008-07-31 13:02:19 UTC (rev 2837) @@ -1,7 +1,9 @@ PROJECT(clucene-core) +#define command line options INCLUDE (DefineOptions) DEFINE_OPTIONS(EXTRA_OPTIONS) +ADD_DEFINITIONS(${EXTRA_OPTIONS} -DMAKE_CLUCENE_CORE_LIB) #add the files to our groups and core SOURCE_GROUP("analysis" ./CLucene/analysis/*) @@ -141,7 +143,6 @@ SET_TARGET_PROPERTIES(clucene-core PROPERTIES VERSION ${CLUCENE_VERSION} SOVERSION ${CLUCENE_SOVERSION} - DEFINE_SYMBOL "MAKE_CLUCENE_CORE_LIB" COMPILE_DEFINITIONS_DEBUG _DEBUG ) @@ -149,15 +150,17 @@ IF ( NOT USE_SHARED_OBJECT_FILES ) IF ( USE_STATIC_SHARED_LIBRARY ) TARGET_LINK_LIBRARIES(clucene-core clucene-shared-static) + ADD_DEFINITIONS(-DLUCENE_SHARED_STATIC}) ELSE ( USE_STATIC_SHARED_LIBRARY ) TARGET_LINK_LIBRARIES(clucene-core clucene-shared) ENDIF ( USE_STATIC_SHARED_LIBRARY ) +ELSE ( NOT USE_SHARED_OBJECT_FILES ) + ADD_DEFINITIONS(-DLUCENE_SHARED_STATIC}) ENDIF ( NOT USE_SHARED_OBJECT_FILES ) SET_TARGET_PROPERTIES(clucene-core-static PROPERTIES VERSION ${CLUCENE_VERSION} SOVERSION ${CLUCENE_SOVERSION} - DEFINE_SYMBOL "MAKE_CLUCENE_CORE_LIB" COMPILE_DEFINITIONS_DEBUG _DEBUG ) @@ -167,22 +170,53 @@ get_filename_component(aname ${file} NAME) file(RELATIVE_PATH relpath ${CMAKE_SOURCE_DIR}/src/core ${apath}) IF ( NOT aname MATCHES "^_.*" ) - install(FILES ${file} DESTINATION include/${relpath}) + install(FILES ${file} + DESTINATION include/${relpath} + COMPONENT development) ENDIF ( NOT aname MATCHES "^_.*" ) ENDFOREACH(file) -#also install CLucene.h -install(FILES ${clucene-shared_BINARY_DIR}/CLucene/clucene-config.h DESTINATION lib/CLucene ) -#and install libraries -install(TARGETS clucene-core DESTINATION lib) -install(TARGETS clucene-core-static DESTINATION lib) +#install clucene-shared headers. +install(FILES ${clucene-shared_SOURCE_DIR}/CLucene/SharedHeader.h + DESTINATION include/CLucene + COMPONENT development) +install(FILES ${clucene-shared_SOURCE_DIR}/CLucene/LuceneThreads.h + DESTINATION include/CLucene + COMPONENT development ) +# code for installing an script to help cmake applications determine +# the CLucene version number +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/CLuceneConfig.cmake" " + set(CLUCENE_VERSION_MAJOR ${CLUCENE_VERSION_MAJOR}) + set(CLUCENE_VERSION_MINOR ${CLUCENE_VERSION_MINOR}) + set(CLUCENE_VERSION_REVISION ${CLUCENE_VERSION_REVISION}) + set(CLUCENE_VERSION_PATCH ${CLUCENE_VERSION_PATCH}) + + set(CLUCENE_VERSION ${CLUCENE_VERSION}) + set(CLUCENE_SOVERSION ${CLUCENE_SOVERSION}) +") -# code for installing an script to help cmake applications determint the Strigi -# version number -#file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/StrigiConfig.cmake" " -#set(STRIGI_VERSION_MAJOR ${STRIGI_VERSION_MAJOR}) -#set(STRIGI_VERSION_MINOR ${STRIGI_VERSION_MINOR}) -#set(STRIGI_VERSION_PATCH ${STRIGI_VERSION_PATCH}) -#") -#install(FILES ${CMAKE_CURRENT_BINARY_DIR}/StrigiConfig.cmake DESTINATION ${LIB_DESTINATION}/strigi) +#install non system-independent +IF ( LUCENE_SYS_INCLUDES ) + install(FILES ${clucene-shared_BINARY_DIR}/CLucene/clucene-config.h + DESTINATION ${LUCENE_SYS_INCLUDES}/CLucene + COMPONENT development) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/CLuceneConfig.cmake + DESTINATION ${LUCENE_SYS_INCLUDES}/CLucene + COMPONENT development) +ELSE ( LUCENE_SYS_INCLUDES ) + install(FILES ${clucene-shared_BINARY_DIR}/CLucene/clucene-config.h + DESTINATION include/CLucene + COMPONENT development) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/CLuceneConfig.cmake + DESTINATION include/CLucene + COMPONENT development) +ENDIF ( LUCENE_SYS_INCLUDES ) + +#and install libraries +install(TARGETS clucene-core + DESTINATION lib + COMPONENT runtime ) +install(TARGETS clucene-core-static + DESTINATION lib + COMPONENT runtime ) Modified: branches/lucene2_3_2/src/demo/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/demo/CMakeLists.txt 2008-07-30 18:23:09 UTC (rev 2836) +++ branches/lucene2_3_2/src/demo/CMakeLists.txt 2008-07-31 13:02:19 UTC (rev 2837) @@ -21,8 +21,8 @@ ${demo_HEADERS} ) - -SET_TARGET_PROPERTIES(cl_demo PROPERTIES - DEFINE_SYMBOL "" -) -TARGET_LINK_LIBRARIES(cl_demo clucene-core) +IF ( USE_STATIC_SHARED_LIBRARY ) + TARGET_LINK_LIBRARIES(cl_demo clucene-core clucene-shared-static) +ELSE ( USE_STATIC_SHARED_LIBRARY ) + TARGET_LINK_LIBRARIES(cl_demo clucene-core clucene-shared) +ENDIF ( USE_STATIC_SHARED_LIBRARY ) Modified: branches/lucene2_3_2/src/shared/CLucene/LuceneThreads.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/LuceneThreads.h 2008-07-30 18:23:09 UTC (rev 2836) +++ branches/lucene2_3_2/src/shared/CLucene/LuceneThreads.h 2008-07-31 13:02:19 UTC (rev 2837) @@ -25,7 +25,7 @@ #else #if defined(_CL_HAVE_PTHREAD) #define _LUCENE_THREADID_TYPE pthread_t - class CLUCENE_EXPORT mutex_thread + class CLUCENE_SHARED_EXPORT mutex_thread { private: struct Internal; @@ -41,7 +41,7 @@ #elif defined(_CL_HAVE_WIN32_THREADS) #define _LUCENE_THREADID_TYPE uint64_t - class CLUCENE_EXPORT mutex_thread + class CLUCENE_SHARED_EXPORT mutex_thread { private: struct Internal; @@ -66,7 +66,7 @@ #endif //don't implement /** @internal */ - class CLUCENE_EXPORT mutexGuard + class CLUCENE_SHARED_EXPORT mutexGuard { private: _LUCENE_THREADMUTEX* mrMutex; Modified: branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-07-30 18:23:09 UTC (rev 2836) +++ branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-07-31 13:02:19 UTC (rev 2837) @@ -119,38 +119,41 @@ #if defined(_WIN32) || defined(_WIN64) #define CLUCENE_EXPORT_DECL __declspec(dllexport) #define CLUCENE_IMPORT_DECL __declspec(dllimport) + #define CLUCENE_LOCAL_DECL #elif defined(_CL_HAVE_GCCVISIBILITYPATCH) #define CLUCENE_EXPORT_DECL __attribute__ ((visibility("default"))) #define CLUCENE_LOCAL_DECL __attribute__ ((visibility("hidden"))) -#endif -#ifndef CLUCENE_EXPORT_DECL + #define CLUCENE_IMPORT_DECL +#else #define CLUCENE_EXPORT_DECL -#endif -#ifndef CLUCENE_IMPORT_DECL #define CLUCENE_IMPORT_DECL -#endif -#ifndef CLUCENE_LOCAL_DECL #define CLUCENE_LOCAL_DECL #endif //define for the libraries -#if defined(MAKE_CLUCENE_CORE_LIB) - #define CLUCENE_EXPORT CLUCENE_EXPORT_DECL +#if defined(clucene_shared_EXPORT) + #define CLUCENE_SHARED_EXPORT CLUCENE_EXPORT_DECL #define CLUCENE_LOCAL CLUCENE_LOCAL_DECL +#elif defined(MAKE_CLUCENE_SHARED_LIB) + #define CLUCENE_SHARED_EXPORT //don't export if we are building a static library #else - #define CLUCENE_EXPORT CLUCENE_IMPORT_DECL + #define CLUCENE_SHARED_EXPORT CLUCENE_IMPORT_DECL #endif -#if defined(MAKE_CLUCENE_CONTRIBS_LIB) - #define CLUCENE_CONTRIBS_EXPORT CLUCENE_EXPORT_DECL +#if defined(clucene_core_EXPORT) + #define CLUCENE_EXPORT CLUCENE_EXPORT_DECL #define CLUCENE_LOCAL CLUCENE_LOCAL_DECL +#elif defined(MAKE_CLUCENE_CORE_LIB) + #define CLUCENE_EXPORT #else - #define CLUCENE_CONTRIBS_EXPORT CLUCENE_IMPORT_DECL + #define CLUCENE_EXPORT CLUCENE_IMPORT_DECL #endif -#if defined(MAKE_CLUCENE_SHARED_LIB) - #define CLUCENE_SHARED_EXPORT CLUCENE_EXPORT_DECL +#if defined(clucene_contribs_EXPORTS) + #define CLUCENE_CONTRIBS_EXPORT CLUCENE_EXPORT_DECL #define CLUCENE_LOCAL CLUCENE_LOCAL_DECL +#elif defined(MAKE_CLUCENE_CONTRIBS_LIB) + #define CLUCENE_CONTRIBS_EXPORT #else - #define CLUCENE_SHARED_EXPORT CLUCENE_IMPORT_DECL + #define CLUCENE_CONTRIBS_EXPORT CLUCENE_IMPORT_DECL #endif #ifndef CLUCENE_LOCAL #define CLUCENE_LOCAL Modified: branches/lucene2_3_2/src/shared/CLucene/_SharedHeader.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/_SharedHeader.h 2008-07-30 18:23:09 UTC (rev 2836) +++ branches/lucene2_3_2/src/shared/CLucene/_SharedHeader.h 2008-07-31 13:02:19 UTC (rev 2837) @@ -35,9 +35,9 @@ ///a blank string... -extern const TCHAR* _LUCENE_BLANK_STRING; +CLUCENE_SHARED_EXPORT extern const TCHAR* _LUCENE_BLANK_STRING; #define LUCENE_BLANK_STRING _LUCENE_BLANK_STRING -extern const char* _LUCENE_BLANK_ASTRING; +CLUCENE_SHARED_EXPORT extern const char* _LUCENE_BLANK_ASTRING; #define LUCENE_BLANK_ASTRING _LUCENE_BLANK_ASTRING #if defined(_WIN32) || defined(_WIN64) Modified: branches/lucene2_3_2/src/shared/CLucene/debug/error.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/debug/error.h 2008-07-30 18:23:09 UTC (rev 2836) +++ branches/lucene2_3_2/src/shared/CLucene/debug/error.h 2008-07-31 13:02:19 UTC (rev 2837) @@ -53,7 +53,7 @@ */ #else - class CLUCENE_EXPORT CLuceneError + class CLUCENE_SHARED_EXPORT CLuceneError { int error_number; char* _awhat; Modified: branches/lucene2_3_2/src/shared/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-07-30 18:23:09 UTC (rev 2836) +++ branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-07-31 13:02:19 UTC (rev 2837) @@ -172,6 +172,7 @@ find_package(Threads REQUIRED) IF ( CMAKE_USE_WIN32_THREADS_INIT ) SET ( _CL_HAVE_WIN32_THREADS 1 ) + SET ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_THREAD_LIBS_INIT}" ) ENDIF ( CMAKE_USE_WIN32_THREADS_INIT ) IF ( CMAKE_USE_PTHREADS_INIT ) SET ( _CL_HAVE_PTHREAD 1 ) @@ -270,22 +271,17 @@ SET_TARGET_PROPERTIES(clucene-shared PROPERTIES VERSION ${CLUCENE_VERSION} SOVERSION ${CLUCENE_SOVERSION} - DEFINE_SYMBOL "MAKE_CLUCENE_SHARED_LIB" COMPILE_DEFINITIONS_DEBUG _DEBUG ) SET_TARGET_PROPERTIES(clucene-shared-static PROPERTIES VERSION ${CLUCENE_VERSION} SOVERSION ${CLUCENE_SOVERSION} - DEFINE_SYMBOL "MAKE_CLUCENE_SHARED_LIB" COMPILE_DEFINITIONS_DEBUG _DEBUG ) -#install public headers. -install(FILES ${clucene-shared_SOURCE_DIR}/CLucene/SharedHeader.h DESTINATION include/CLucene ) -install(FILES ${clucene-shared_SOURCE_DIR}/CLucene/LuceneThreads.h DESTINATION include/CLucene ) - -IF ( NOT USE_SHARED_OBJECT_FILES ) - IF ( NOT USE_STATIC_SHARED_LIBRARY ) - install(TARGETS clucene-shared DESTINATION lib) - ENDIF ( NOT USE_STATIC_SHARED_LIBRARY ) -ENDIF ( NOT USE_SHARED_OBJECT_FILES ) +install(TARGETS clucene-shared + DESTINATION lib + COMPONENT development ) +install(TARGETS clucene-shared-static + DESTINATION lib + COMPONENT runtime ) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <syn...@us...> - 2008-08-04 13:02:36
|
Revision: 2856 http://clucene.svn.sourceforge.net/clucene/?rev=2856&view=rev Author: synhershko Date: 2008-08-04 13:02:32 +0000 (Mon, 04 Aug 2008) Log Message: ----------- Fixes TermVectorsReader and Windows compilation Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/index/TermVectorReader.cpp branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h Modified: branches/lucene2_3_2/src/core/CLucene/index/TermVectorReader.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/index/TermVectorReader.cpp 2008-08-03 23:24:00 UTC (rev 2855) +++ branches/lucene2_3_2/src/core/CLucene/index/TermVectorReader.cpp 2008-08-04 13:02:32 UTC (rev 2856) @@ -103,6 +103,7 @@ tvfFormat = copy.tvfFormat; _size = copy._size; fieldInfos = copy.fieldInfos; + docStoreOffset = copy.docStoreOffset; } TermVectorsReader* TermVectorsReader::clone() const{ if (tvx == NULL || tvd == NULL || tvf == NULL) Modified: branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-08-03 23:24:00 UTC (rev 2855) +++ branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-08-04 13:02:32 UTC (rev 2856) @@ -160,7 +160,7 @@ #endif //inline definitions -#ifdef __MINGW32__ +#if defined(__MINGW32__) || defined(_MSC_VER) #define CLUCENE_SHARED_INLINE_EXPORT #define CLUCENE_INLINE_EXPORT #define CLUCENE_CONTRIBS_INLINE_EXPORT This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-08-15 12:44:11
|
Revision: 2859 http://clucene.svn.sourceforge.net/clucene/?rev=2859&view=rev Author: ustramooner Date: 2008-08-15 12:44:08 +0000 (Fri, 15 Aug 2008) Log Message: ----------- removed NO_* query exclusion macros Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h branches/lucene2_3_2/src/test/queryParser/TestQueryParser.cpp Modified: branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h 2008-08-15 12:41:14 UTC (rev 2858) +++ branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h 2008-08-15 12:44:08 UTC (rev 2859) @@ -289,18 +289,10 @@ //these functions may be defined under certain compilation conditions. //note that this functionality is deprecated, you should create your own queryparser //if you want to remove this functionality...it will be removed... be warned! -#ifdef NO_PREFIX_QUERY virtual CL_NS(search)::Query* GetPrefixQuery(const TCHAR* field,const TCHAR* termStr){ return NULL; } -#endif -#ifdef NO_FUZZY_QUERY virtual CL_NS(search)::Query* GetFuzzyQuery(const TCHAR* field,const TCHAR* termStr){ return NULL; } -#endif -#ifdef NO_RANGE_QUERY virtual CL_NS(search)::Query* GetRangeQuery(const TCHAR* field, const TCHAR* part1, const TCHAR* part2, bool inclusive) { return NULL; } -#endif -#ifdef NO_WILDCARD_QUERY virtual CL_NS(search)::Query* GetWildcardQuery(const TCHAR* field, TCHAR* termStr) { return NULL; } -#endif private: /** * matches for CONJUNCTION Modified: branches/lucene2_3_2/src/test/queryParser/TestQueryParser.cpp =================================================================== --- branches/lucene2_3_2/src/test/queryParser/TestQueryParser.cpp 2008-08-15 12:41:14 UTC (rev 2858) +++ branches/lucene2_3_2/src/test/queryParser/TestQueryParser.cpp 2008-08-15 12:44:08 UTC (rev 2859) @@ -108,8 +108,8 @@ assertQueryEquals(tc,_T("term term term"), NULL, _T("term term term")); #ifdef _UCS2 - TCHAR tmp1[100]; - + TCHAR tmp1[100]; + lucene_utf8towcs(tmp1,"t\xc3\xbcrm term term",100); assertQueryEquals(tc,tmp1, NULL, tmp1); assertQueryEquals(tc,tmp1, &a, tmp1); @@ -223,12 +223,12 @@ assertQueryEquals(tc,_T("term*"), NULL, _T("term*")); assertQueryEquals(tc,_T("term*^2"), NULL, _T("term*^2.0")); -#ifndef NO_FUZZY_QUERY assertQueryEquals(tc,_T("term~"), NULL, _T("term~0.5")); + assertQueryEquals(tc,_T("term~0.5"), NULL, _T("term")); assertQueryEquals(tc,_T("term~^2"), NULL, _T("term^2.0~0.5")); assertQueryEquals(tc,_T("term^2~"), NULL, _T("term^2.0~0.5")); assertTrue(tc, _T("term~"), NULL,_T("FuzzyQuery"), _T("term~0.5")); -#endif + assertQueryEquals(tc,_T("term*germ"), NULL, _T("term*germ")); assertQueryEquals(tc,_T("term*germ^3"), NULL, _T("term*germ^3.0")); @@ -319,15 +319,10 @@ SUITE_ADD_TEST(suite, testNumber); SUITE_ADD_TEST(suite, testPunct); -#ifndef NO_FUZZY_QUERY SUITE_ADD_TEST(suite, testSlop); -#endif -#ifndef NO_RANGE_QUERY SUITE_ADD_TEST(suite, testRange); -#endif -#ifndef NO_WILDCARD_QUERY SUITE_ADD_TEST(suite, testWildcard); -#endif + return suite; } // EOF This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-08-15 13:13:20
|
Revision: 2863 http://clucene.svn.sourceforge.net/clucene/?rev=2863&view=rev Author: ustramooner Date: 2008-08-15 13:13:15 +0000 (Fri, 15 Aug 2008) Log Message: ----------- using cl_stat_t instead of fileStat for the struct's now. this was clashing on bcc and use cl_open instead of _open. was clashing on bcc Modified Paths: -------------- branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp branches/lucene2_3_2/src/shared/CLucene/util/dirent.cpp branches/lucene2_3_2/src/test/analysis/TestAnalysis.cpp branches/lucene2_3_2/src/test/index/TestReuters.cpp Modified: branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp 2008-08-15 13:11:53 UTC (rev 2862) +++ branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp 2008-08-15 13:13:15 UTC (rev 2863) @@ -55,7 +55,7 @@ //exist on non-msvc platform, so lets put it here int64_t Misc::filelength(int filehandle) { - struct fileStat info; + struct cl_stat_t info; if (fileHandleStat(filehandle, &info) == -1) _CLTHROWA( CL_ERR_IO,"fileStat error" ); return info.st_size; @@ -174,13 +174,13 @@ bool Misc::dir_Exists(const char* path){ if ( !path || !*path ) return false; - struct fileStat buf; + struct cl_stat_t buf; int32_t ret = fileStat(path,&buf); return ( ret == 0); } int64_t Misc::file_Size(const char* path){ - struct fileStat buf; + struct cl_stat_t buf; if ( fileStat(path,&buf) == 0 ) return buf.st_size; else Modified: branches/lucene2_3_2/src/shared/CLucene/util/dirent.cpp =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/util/dirent.cpp 2008-08-15 13:11:53 UTC (rev 2862) +++ branches/lucene2_3_2/src/shared/CLucene/util/dirent.cpp 2008-08-15 13:13:15 UTC (rev 2863) @@ -33,7 +33,7 @@ } /* Attempt to determine if the given path really is a directory. */ - struct fileStat rcs; + struct cl_stat_t rcs; if ( fileStat(szPath,&rcs) == -1) { /* call GetLastError for more error info */ @@ -174,7 +174,7 @@ if (!bThisFolderOrUpFolder) { - struct fileStat buf; + struct cl_stat_t buf; char buffer[CL_MAX_DIR]; size_t bl = strlen(dirp->dd_name)-strlen(DIRENT_SEARCH_SUFFIX); strncpy(buffer,dirp->dd_name,bl); Modified: branches/lucene2_3_2/src/test/analysis/TestAnalysis.cpp =================================================================== --- branches/lucene2_3_2/src/test/analysis/TestAnalysis.cpp 2008-08-15 13:11:53 UTC (rev 2862) +++ branches/lucene2_3_2/src/test/analysis/TestAnalysis.cpp 2008-08-15 13:13:15 UTC (rev 2863) @@ -34,7 +34,7 @@ /*todo: move this to contribs because we have no filereader void _testFile(CuTest *tc,const char* fname, bool verbose) { - struct fileStat buf; + struct cl_stat_t buf; fileStat(fname,&buf); int64_t bytes = buf.st_size; Modified: branches/lucene2_3_2/src/test/index/TestReuters.cpp =================================================================== --- branches/lucene2_3_2/src/test/index/TestReuters.cpp 2008-08-15 13:11:53 UTC (rev 2862) +++ branches/lucene2_3_2/src/test/index/TestReuters.cpp 2008-08-15 13:13:15 UTC (rev 2863) @@ -66,7 +66,7 @@ writer.setMaxFieldLength(10000); DIR* srcdir = opendir(reuters_srcdirectory); struct dirent* fl = readdir(srcdir); - struct fileStat buf; + struct cl_stat_t buf; char tmppath[CL_MAX_DIR]; strncpy(tmppath,reuters_srcdirectory,CL_MAX_DIR); strcat(tmppath,"/"); This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-08-15 13:15:23
|
Revision: 2867 http://clucene.svn.sourceforge.net/clucene/?rev=2867&view=rev Author: ustramooner Date: 2008-08-15 13:15:17 +0000 (Fri, 15 Aug 2008) Log Message: ----------- various bcc fixes Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/util/ThreadLocal.cpp branches/lucene2_3_2/src/shared/CLucene/_clucene-config.h.cmake branches/lucene2_3_2/src/shared/CLucene/clucene-config.h.cmake branches/lucene2_3_2/src/shared/CLucene/config/_threads.h Modified: branches/lucene2_3_2/src/core/CLucene/util/ThreadLocal.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/ThreadLocal.cpp 2008-08-15 13:14:28 UTC (rev 2866) +++ branches/lucene2_3_2/src/core/CLucene/util/ThreadLocal.cpp 2008-08-15 13:15:17 UTC (rev 2867) @@ -13,14 +13,14 @@ //todo: using http://en.wikipedia.org/wiki/Thread-local_storage#Pthreads_implementation //would work better... but lots of testing would be needed first... -typedef CL_NS(util)::CLSetList<_ThreadLocal::ShutdownHook*, - CL_NS(util)::Compare::Void<_ThreadLocal::ShutdownHook>, - CL_NS(util)::Deletor::ConstNullVal<_ThreadLocal::ShutdownHook*> > ShutdownHooksType; - typedef CL_NS(util)::CLMultiMap<_LUCENE_THREADID_TYPE, _ThreadLocal*, CL_NS(util)::CLuceneThreadIdCompare, CL_NS(util)::Deletor::ConstNullVal<_LUCENE_THREADID_TYPE>, CL_NS(util)::Deletor::ConstNullVal<_ThreadLocal*> > ThreadLocalsType; + +typedef CL_NS(util)::CLSetList<_ThreadLocal::ShutdownHook*, + CL_NS(util)::Compare::Void<_ThreadLocal::ShutdownHook>, + CL_NS(util)::Deletor::ConstNullVal<_ThreadLocal::ShutdownHook*> > ShutdownHooksType; #ifdef _LUCENE_THREADMUTEX //the lock for locking ThreadLocalBase_threadLocals Modified: branches/lucene2_3_2/src/shared/CLucene/_clucene-config.h.cmake =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/_clucene-config.h.cmake 2008-08-15 13:14:28 UTC (rev 2866) +++ branches/lucene2_3_2/src/shared/CLucene/_clucene-config.h.cmake 2008-08-15 13:15:17 UTC (rev 2867) @@ -56,6 +56,7 @@ #define _ILONGLONG(x) ${_CL_ILONGLONG_VALUE} ${FUNCTION_FILESTAT} +${TYPE_CL_STAT_T} ${FUNCTION_FILESIZE} ${FUNCTION_FILESEEK} ${FUNCTION_FILETELL} @@ -64,7 +65,7 @@ ${FUNCTION__RENAME} ${FUNCTION__CLOSE} ${FUNCTION__READ} -${FUNCTION__OPEN} +${FUNCTION__CL_OPEN} ${FUNCTION__WRITE} ${FUNCTION__SNPRINTF} ${FUNCTION__MKDIR} @@ -102,10 +103,12 @@ //not sure why, but cygwin reports _S_IREAD, but doesn't actually work... //TODO: make this work properly (this bit shouldn't be necessary) #ifdef __CYGWIN__ - #define _ftime ftime - #define _S_IREAD 0333 #define _S_IWRITE 0333 #endif +#ifdef __BORLANDC__ //borland compiler + #define O_RANDOM 0 #endif + +#endif Modified: branches/lucene2_3_2/src/shared/CLucene/clucene-config.h.cmake =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/clucene-config.h.cmake 2008-08-15 13:14:28 UTC (rev 2866) +++ branches/lucene2_3_2/src/shared/CLucene/clucene-config.h.cmake 2008-08-15 13:15:17 UTC (rev 2867) @@ -128,4 +128,11 @@ #cmakedefine _CL_DISABLE_MULTITHREADING +#ifdef __BORLANDC__ //borland compiler + //todo: bcc incorrectly detects this... fix this in cmake + #undef LUCENE_STATIC_CONSTANT + #define LUCENE_STATIC_CONSTANT(type, assignment) enum { assignment } #endif + + +#endif Modified: branches/lucene2_3_2/src/shared/CLucene/config/_threads.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/config/_threads.h 2008-08-15 13:14:28 UTC (rev 2866) +++ branches/lucene2_3_2/src/shared/CLucene/config/_threads.h 2008-08-15 13:15:17 UTC (rev 2867) @@ -12,8 +12,9 @@ //do nothing #elif defined(_CL_HAVE_WIN32_THREADS) //we have not explicity included windows.h and windows.h has - //not been included (check _WINDOWS_), then we must define + //not been included (check _WINBASE_), then we must define //our own definitions to the thread locking functions: + #ifndef _WINBASE_ extern "C"{ struct CRITICAL_SECTION { @@ -30,6 +31,7 @@ __declspec(dllimport) void __stdcall DeleteCriticalSection(CRITICAL_SECTION *); __declspec(dllimport) unsigned long __stdcall GetCurrentThreadId(); } + #endif //_WINBASE_ #elif defined(_CL_HAVE_PTHREAD) #include <pthread.h> #endif This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-08-18 16:52:30
|
Revision: 2870 http://clucene.svn.sourceforge.net/clucene/?rev=2870&view=rev Author: ustramooner Date: 2008-08-18 16:52:18 +0000 (Mon, 18 Aug 2008) Log Message: ----------- fix queryparser deprecation of NO_* macros. more fuzzy query tests Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h branches/lucene2_3_2/src/test/queryParser/TestQueryParser.cpp Modified: branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h 2008-08-17 17:18:51 UTC (rev 2869) +++ branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h 2008-08-18 16:52:18 UTC (rev 2870) @@ -285,14 +285,6 @@ //deprecated functions _CL_DEPRECATED( setLowercaseExpandedTerms ) void setLowercaseWildcardTerms(bool lowercaseWildcardTerms); _CL_DEPRECATED( getLowercaseExpandedTerms ) bool getLowercaseWildcardTerms() const; -protected: - //these functions may be defined under certain compilation conditions. - //note that this functionality is deprecated, you should create your own queryparser - //if you want to remove this functionality...it will be removed... be warned! - virtual CL_NS(search)::Query* GetPrefixQuery(const TCHAR* field,const TCHAR* termStr){ return NULL; } - virtual CL_NS(search)::Query* GetFuzzyQuery(const TCHAR* field,const TCHAR* termStr){ return NULL; } - virtual CL_NS(search)::Query* GetRangeQuery(const TCHAR* field, const TCHAR* part1, const TCHAR* part2, bool inclusive) { return NULL; } - virtual CL_NS(search)::Query* GetWildcardQuery(const TCHAR* field, TCHAR* termStr) { return NULL; } private: /** * matches for CONJUNCTION Modified: branches/lucene2_3_2/src/test/queryParser/TestQueryParser.cpp =================================================================== --- branches/lucene2_3_2/src/test/queryParser/TestQueryParser.cpp 2008-08-17 17:18:51 UTC (rev 2869) +++ branches/lucene2_3_2/src/test/queryParser/TestQueryParser.cpp 2008-08-18 16:52:18 UTC (rev 2870) @@ -202,6 +202,9 @@ assertQueryEquals(tc,_T("\"term germ\"~2"), NULL, _T("\"term germ\"~2") ); assertQueryEquals(tc,_T("\"term germ\"~2 flork"), NULL, _T("\"term germ\"~2 flork") ); assertQueryEquals(tc,_T("\"term\"~2"), NULL, _T("term")); + assertQueryEquals(tc,_T("term~2"), NULL, _T("term")); + assertQueryEquals(tc,_T("term~0.5"), NULL, _T("term")); + assertQueryEquals(tc,_T("term~0.6"), NULL, _T("term")); assertQueryEquals(tc,_T("\" \"~2 germ"), NULL, _T("germ")); assertQueryEquals(tc,_T("\"term germ\"~2^2"), NULL, _T("\"term germ\"~2^2.0") ); } This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-08-22 08:54:41
|
Revision: 2872 http://clucene.svn.sourceforge.net/clucene/?rev=2872&view=rev Author: ustramooner Date: 2008-08-22 08:54:36 +0000 (Fri, 22 Aug 2008) Log Message: ----------- fixes for cygwin, msvc and bcc Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/store/LockFactory.cpp branches/lucene2_3_2/src/shared/cmake/MacroChooseSymbol.cmake branches/lucene2_3_2/src/shared/cmake/MacroChooseType.cmake Modified: branches/lucene2_3_2/src/core/CLucene/store/LockFactory.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/store/LockFactory.cpp 2008-08-18 16:54:13 UTC (rev 2871) +++ branches/lucene2_3_2/src/core/CLucene/store/LockFactory.cpp 2008-08-22 08:54:36 UTC (rev 2872) @@ -105,7 +105,7 @@ FSLockFactory::~FSLockFactory() { - _CLDELETE( lockDir ); + _CLDELETE_CaARRAY( lockDir ); } void FSLockFactory::setLockDir( const char* lockDir ) Modified: branches/lucene2_3_2/src/shared/cmake/MacroChooseSymbol.cmake =================================================================== --- branches/lucene2_3_2/src/shared/cmake/MacroChooseSymbol.cmake 2008-08-18 16:54:13 UTC (rev 2871) +++ branches/lucene2_3_2/src/shared/cmake/MacroChooseSymbol.cmake 2008-08-22 08:54:36 UTC (rev 2872) @@ -3,9 +3,14 @@ INCLUDE (Macro_ChooseStatus) MACRO(CHOOSE_SYMBOL name options) - IF ( HAVE_WINDOWS_H ) - SET (CHOOSE_SYMBOL_INCLUDES "${CHOOSE_SYMBOL_INCLUDES};windows.h") - ENDIF ( HAVE_WINDOWS_H ) + #note: don't add windows.h to this list, since we don't want to find things + #in there (bcc might make you think otherwise, but it's true!)... + + IF ( CYGWIN ) + #cygwin requires this to determine some things... + SET ( CMAKE_REQUIRED_DEFINITIONS "-D_WIN32") + ENDIF( CYGWIN ) + IF ( HAVE_LIMITS_H ) SET (CHOOSE_SYMBOL_INCLUDES "${CHOOSE_SYMBOL_INCLUDES};limits.h") ENDIF ( HAVE_LIMITS_H ) @@ -53,4 +58,5 @@ ENDIF ( NOT SYMBOL_${NAME} ) SET (CHOOSE_SYMBOL_INCLUDES) + SET (CMAKE_REQUIRED_DEFINITIONS) ENDMACRO(CHOOSE_SYMBOL) Modified: branches/lucene2_3_2/src/shared/cmake/MacroChooseType.cmake =================================================================== --- branches/lucene2_3_2/src/shared/cmake/MacroChooseType.cmake 2008-08-18 16:54:13 UTC (rev 2871) +++ branches/lucene2_3_2/src/shared/cmake/MacroChooseType.cmake 2008-08-22 08:54:36 UTC (rev 2872) @@ -4,12 +4,11 @@ #macro that sets OUTPUT as the value of oneof options (if _CL_HAVE_OPTION exists) MACRO(CHOOSE_TYPE name size sign options) + #note: don't add windows.h to this list, since we don't want to find things + #in there (bcc might make you think otherwise, but it's true!)... IF ( HAVE_TCHAR_H ) SET (CMAKE_EXTRA_INCLUDE_FILES "${CMAKE_EXTRA_INCLUDE_FILES};tchar.h") ENDIF ( HAVE_TCHAR_H ) - IF ( HAVE_WINDOWS_H ) - SET (CMAKE_EXTRA_INCLUDE_FILES "${CMAKE_EXTRA_INCLUDE_FILES};windows.h") - ENDIF ( HAVE_WINDOWS_H ) STRING(TOUPPER ${name} NAME) FOREACH(option ${options}) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <syn...@us...> - 2008-08-26 14:15:19
|
Revision: 2877 http://clucene.svn.sourceforge.net/clucene/?rev=2877&view=rev Author: synhershko Date: 2008-08-26 14:15:15 +0000 (Tue, 26 Aug 2008) Log Message: ----------- More several more updates, required for FieldsReader and TermInfosReader (upcoming) Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/files_list.txt branches/lucene2_3_2/src/core/CLucene/index/FieldInfos.cpp branches/lucene2_3_2/src/core/CLucene/index/Terms.h branches/lucene2_3_2/src/core/CLucene/index/_FieldInfos.h branches/lucene2_3_2/src/shared/CLucene/debug/error.h Modified: branches/lucene2_3_2/src/core/CLucene/index/FieldInfos.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/index/FieldInfos.cpp 2008-08-26 11:52:28 UTC (rev 2876) +++ branches/lucene2_3_2/src/core/CLucene/index/FieldInfos.cpp 2008-08-26 14:15:15 UTC (rev 2877) @@ -44,6 +44,11 @@ CL_NS(util)::CLStringIntern::unintern(name); } +FieldInfo* FieldInfo::clone() { + return _CLNEW FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector, + storeOffsetWithTermVector, omitNorms, storePayloads); +} + FieldInfos::FieldInfos(): byName(false,false),byNumber(true) { } @@ -65,18 +70,52 @@ ); } +FieldInfos* FieldInfos::clone() +{ + FieldInfos* fis = _CLNEW FieldInfos(); + const size_t numField = byNumber.size(); + for(size_t i=0;i<numField;i++) { + FieldInfo* fi = byNumber[i]->clone(); + fis->byNumber.push_back(fi); + fis->byName.put( fi->name, fi); + } + return fis; +} + void FieldInfos::add(const Document* doc) { DocumentFieldEnumeration* fields = doc->getFields(); Field* field; while (fields->hasMoreElements()) { field = fields->nextElement(); - add(field->name(), field->isIndexed(), field->isTermVectorStored()); + add(field->name(), field->isIndexed(), field->isTermVectorStored(), field->isStorePositionWithTermVector(), + field->isStoreOffsetWithTermVector(), field->getOmitNorms()); } _CLDELETE(fields); } +void FieldInfos::addIndexed(const TCHAR** names, const bool storeTermVectors, const bool storePositionWithTermVector, + const bool storeOffsetWithTermVector) { + size_t i = 0; + while (names[i]) { + add(names[i], true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector); + ++i; + } +} + +void FieldInfos::add(const TCHAR** names,const bool isIndexed, const bool storeTermVectors, + const bool storePositionWithTermVector, const bool storeOffsetWithTermVector, const bool omitNorms, const bool storePayloads) +{ + size_t i=0; + while ( names[i] != NULL ){ + add(names[i], isIndexed, storeTermVectors, storePositionWithTermVector, + storeOffsetWithTermVector, omitNorms, storePayloads); + ++i; + } +} + FieldInfo* FieldInfos::add( const TCHAR* name, const bool isIndexed, const bool storeTermVector, - const bool storePositionWithTermVector, const bool storeOffsetWithTermVector, const bool omitNorms, const bool storePayloads) { + const bool storePositionWithTermVector, const bool storeOffsetWithTermVector, const bool omitNorms, + const bool storePayloads) { FieldInfo* fi = fieldInfo(name); if (fi == NULL) { return addInternal(name, isIndexed, storeTermVector, @@ -105,28 +144,28 @@ return fi; } -void FieldInfos::add(const TCHAR** names,const bool isIndexed, const bool storeTermVectors, - const bool storePositionWithTermVector, const bool storeOffsetWithTermVector, const bool omitNorms, const bool storePayloads) -{ - int32_t i=0; - while ( names[i] != NULL ){ - add(names[i], isIndexed, storeTermVectors, storePositionWithTermVector, - storeOffsetWithTermVector, omitNorms, storePayloads); - ++i; - } +FieldInfo* FieldInfos::addInternal( const TCHAR* name, const bool isIndexed, const bool storeTermVector, + const bool storePositionWithTermVector, const bool storeOffsetWithTermVector, + const bool omitNorms, const bool storePayloads) { + + FieldInfo* fi = _CLNEW FieldInfo(name, isIndexed, byNumber.size(), storeTermVector, + storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads); + byNumber.push_back(fi); + byName.put( fi->name, fi); + return fi; } int32_t FieldInfos::fieldNumber(const TCHAR* fieldName)const { FieldInfo* fi = fieldInfo(fieldName); - return (fi!=NULL)?fi->number:-1; + return (fi!=NULL) ? fi->number : -1; } - FieldInfo* FieldInfos::fieldInfo(const TCHAR* fieldName) const { FieldInfo* ret = byName.get(fieldName); return ret; } -const TCHAR* FieldInfos::fieldName(const int32_t fieldNumber)const { + +const TCHAR* FieldInfos::fieldName(const int32_t fieldNumber) const { FieldInfo* fi = fieldInfo(fieldNumber); return (fi==NULL)?LUCENE_BLANK_STRING:fi->name; } @@ -137,10 +176,18 @@ return byNumber[fieldNumber]; } -int32_t FieldInfos::size()const { +size_t FieldInfos::size()const { return byNumber.size(); } +bool FieldInfos::hasVectors() const{ + for (size_t i = 0; i < size(); i++) { + if (fieldInfo(i)->storeTermVector) + return true; + } + return false; +} + void FieldInfos::write(Directory* d, const char* name) const{ IndexOutput* output = d->createOutput(name); try { @@ -152,10 +199,10 @@ } void FieldInfos::write(IndexOutput* output) const{ - output->writeVInt(size()); + output->writeVInt(static_cast<int32_t>(size())); FieldInfo* fi; uint8_t bits; - for (int32_t i = 0; i < size(); ++i) { + for (size_t i = 0; i < size(); ++i) { fi = fieldInfo(i); bits = 0x0; if (fi->isIndexed) bits |= IS_INDEXED; @@ -171,7 +218,7 @@ } void FieldInfos::read(IndexInput* input) { - int32_t size = input->readVInt(); + int32_t size = input->readVInt();//read in the size uint8_t bits; bool isIndexed,storeTermVector,storePositionsWithTermVector,storeOffsetWithTermVector,omitNorms,storePayloads; for (int32_t i = 0; i < size; ++i){ @@ -188,33 +235,5 @@ _CLDELETE_CARRAY(name); } } -FieldInfo* FieldInfos::addInternal( const TCHAR* name, const bool isIndexed, const bool storeTermVector, - const bool storePositionWithTermVector, const bool storeOffsetWithTermVector, const bool omitNorms, const bool storePayloads) { - FieldInfo* fi = _CLNEW FieldInfo(name, isIndexed, byNumber.size(), storeTermVector, - storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads); - byNumber.push_back(fi); - byName.put( fi->name, fi); - return fi; -} -bool FieldInfos::hasVectors() const{ - for (int32_t i = 0; i < size(); i++) { - if (fieldInfo(i)->storeTermVector) - return true; - } - return false; -} - -FieldInfos* FieldInfos::clone() -{ - FieldInfos* fis = _CLNEW FieldInfos(); - const size_t numField = byNumber.size(); - for(size_t i=0;i<numField;i++) { - FieldInfo* fi = byNumber[i]->clone(); - fis->byNumber.push_back(fi); - fis->byName.put( fi->name, fi); - } - return fis; -} - CL_NS_END Modified: branches/lucene2_3_2/src/core/CLucene/index/Terms.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/index/Terms.h 2008-08-26 11:52:28 UTC (rev 2876) +++ branches/lucene2_3_2/src/core/CLucene/index/Terms.h 2008-08-26 14:15:15 UTC (rev 2877) @@ -83,25 +83,26 @@ }; -// Abstract class for enumerating terms. -// -//<p>Term enumerations are always ordered by Term.compareTo(). Each term in -//the enumeration is greater than all that precede it. +/** Abstract class for enumerating terms. + + <p>Term enumerations are always ordered by Term.compareTo(). Each term in + the enumeration is greater than all that precede it. +*/ class CLUCENE_EXPORT TermEnum: LUCENE_BASE { public: - // Increments the enumeration to the next element. True if one exists. + /** Increments the enumeration to the next element. True if one exists.*/ virtual bool next()=0; // Returns a pointer to the current Term in the enumeration. virtual Term* term()=0; - - // Returns the current Term in the enumeration. + + /** Returns the current Term in the enumeration.*/ virtual Term* term(bool pointer); - // Returns the docFreq of the current Term in the enumeration. + /** Returns the docFreq of the current Term in the enumeration.*/ virtual int32_t docFreq() const=0; - // Closes the enumeration to further activity, freeing resources. + /** Closes the enumeration to further activity, freeing resources. */ virtual void close() =0; virtual ~TermEnum(); Modified: branches/lucene2_3_2/src/core/CLucene/index/_FieldInfos.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/index/_FieldInfos.h 2008-08-26 11:52:28 UTC (rev 2876) +++ branches/lucene2_3_2/src/core/CLucene/index/_FieldInfos.h 2008-08-26 14:15:15 UTC (rev 2877) @@ -61,10 +61,10 @@ //Post - The instance has been destroyed ~FieldInfo(); - FieldInfo* clone() { - return _CLNEW FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector, - storeOffsetWithTermVector, omitNorms, storePayloads); - } + /* Clones this + * @memory - caller is responsible for deleting the returned object + */ + FieldInfo* clone(); }; /** Access to the Fieldable Info file that describes document fields and whether or @@ -100,44 +100,40 @@ * @param d The directory to open the IndexInput from * @param name The name of the file to open the IndexInput from in the Directory * @throws IOException - * - * @see #read */ FieldInfos(CL_NS(store)::Directory* d, const char* name); - int32_t fieldNumber(const TCHAR* fieldName)const; - /** - * Return the fieldinfo object referenced by the fieldNumber. - * @param fieldNumber - * @return the FieldInfo object or null when the given fieldNumber - * doesn't exist. - */ - FieldInfo* fieldInfo(const TCHAR* fieldName) const; - + * Returns a deep clone of this FieldInfos instance. + * @memory caller is responisble for deleting returned object + */ + FieldInfos* clone(); + + /** Adds field info for a Document. */ + void add(const CL_NS(document)::Document* doc); + /** - * Return the fieldName identified by its number. + * Add fields that are indexed. Whether they have termvectors has to be specified. * - * @param fieldNumber - * @return the fieldName or an empty string when the field - * with the given number doesn't exist. - */ - const TCHAR* fieldName(const int32_t fieldNumber)const; + * @param names The names of the fields. An array of TCHARs, last item has to be NULL + * @param storeTermVectors Whether the fields store term vectors or not + * @param storePositionWithTermVector treu if positions should be stored. + * @param storeOffsetWithTermVector true if offsets should be stored + */ + void addIndexed(const TCHAR** names, const bool storeTermVectors, const bool storePositionWithTermVector, const bool storeOffsetWithTermVector); - FieldInfo* fieldInfo(const int32_t fieldNumber) const; - - int32_t size()const; - - bool hasVectors() const; - /** - * Returns a deep clone of this FieldInfos instance. + * Assumes the fields are not storing term vectors. + * + * @param names The names of the fields + * @param isIndexed Whether the fields are indexed or not + * + * @see #add(TCHAR*, bool) */ - FieldInfos* clone(); + void add(const TCHAR** names, const bool isIndexed, const bool storeTermVector=false, + const bool storePositionWithTermVector=false, const bool storeOffsetWithTermVector=false, + const bool omitNorms=false, const bool storePayloads=false); - // Adds field info for a Document. - void add(const CL_NS(document)::Document* doc); - // Merges in information from another FieldInfos. void add(FieldInfos* other); @@ -156,26 +152,47 @@ */ FieldInfo* add(const TCHAR* name, const bool isIndexed, const bool storeTermVector=false, const bool storePositionWithTermVector=false, const bool storeOffsetWithTermVector=false, const bool omitNorms=false, const bool storePayloads=false); + + // was void + FieldInfo* addInternal( const TCHAR* name,const bool isIndexed, const bool storeTermVector, + const bool storePositionWithTermVector, const bool storeOffsetWithTermVector, const bool omitNorms, const bool storePayloads); + + int32_t fieldNumber(const TCHAR* fieldName)const; /** - * Assumes the fields are not storing term vectors - * @param names The names of the fields - * @param isIndexed true if the field is indexed - * @param storeTermVector true if the term vector should be stored + * Return the fieldinfo object referenced by the fieldNumber. + * @param fieldNumber + * @return the FieldInfo object or null when the given fieldNumber + * doesn't exist. + */ + FieldInfo* fieldInfo(const TCHAR* fieldName) const; + + /** + * Return the fieldName identified by its number. * - * @see #add(String, boolean) - */ - void add(const TCHAR** names, const bool isIndexed, const bool storeTermVector=false, - const bool storePositionWithTermVector=false, const bool storeOffsetWithTermVector=false, const bool omitNorms=false, const bool storePayloads=false); + * @param fieldNumber + * @return the fieldName or an empty string when the field + * with the given number doesn't exist. + */ + const TCHAR* fieldName(const int32_t fieldNumber)const; + /** + * Return the fieldinfo object referenced by the fieldNumber. + * @param fieldNumber + * @return the FieldInfo object or null when the given fieldNumber + * doesn't exist. + */ + FieldInfo* fieldInfo(const int32_t fieldNumber) const; + + size_t size()const; + bool hasVectors() const; + + void write(CL_NS(store)::Directory* d, const char* name) const; void write(CL_NS(store)::IndexOutput* output) const; private: void read(CL_NS(store)::IndexInput* input); - // was void - FieldInfo* addInternal( const TCHAR* name,const bool isIndexed, const bool storeTermVector, - const bool storePositionWithTermVector, const bool storeOffsetWithTermVector, const bool omitNorms, const bool storePayloads); }; CL_NS_END Modified: branches/lucene2_3_2/src/shared/CLucene/debug/error.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/debug/error.h 2008-08-26 11:52:28 UTC (rev 2876) +++ branches/lucene2_3_2/src/shared/CLucene/debug/error.h 2008-08-26 14:15:15 UTC (rev 2877) @@ -21,7 +21,7 @@ #define CL_ERR_TooManyClauses 10 #define CL_ERR_RAMTransaction 11 #define CL_ERR_InvalidCast 12 -#define CL_ERR_IllegalState 13 +#define CL_ERR_IllegalState 13 // Sub-error: AlreadyClosed #define CL_ERR_UnknownOperator 14 #define CL_ERR_ConcurrentModification 15 #define CL_ERR_CorruptIndex 16 This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <syn...@us...> - 2008-08-28 13:22:11
|
Revision: 2880 http://clucene.svn.sourceforge.net/clucene/?rev=2880&view=rev Author: synhershko Date: 2008-08-28 13:22:06 +0000 (Thu, 28 Aug 2008) Log Message: ----------- Removed use of Internal from Field to allow for more flexible derivation from it and since it is no longer important to have since void* is used for storing values Some Field optimizations More code porting in FieldsReader Added TestSearch for FSDirectory Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/document/Field.cpp branches/lucene2_3_2/src/core/CLucene/document/Field.h branches/lucene2_3_2/src/core/CLucene/index/FieldsReader.cpp branches/lucene2_3_2/src/core/CLucene/index/_FieldsReader.h branches/lucene2_3_2/src/test/search/TestSearch.cpp Modified: branches/lucene2_3_2/src/core/CLucene/document/Field.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/document/Field.cpp 2008-08-27 12:53:20 UTC (rev 2879) +++ branches/lucene2_3_2/src/core/CLucene/document/Field.cpp 2008-08-28 13:22:06 UTC (rev 2880) @@ -13,38 +13,39 @@ CL_NS_USE(util) CL_NS_DEF(document) +/* struct Field::Internal{ - const TCHAR* _name; + //const TCHAR* _name; //TCHAR* _stringValue; //CL_NS(util)::Reader* _readerValue; //jstreams::StreamBase<char>* _streamValue; //void* fieldsData; - uint32_t config; - float_t boost; -}; + //uint32_t config; + //float_t boost; +};*/ Field::Field(const TCHAR* Name, Reader* reader, int config): - _internal(new Internal), lazy(false) + /*_internal(new Internal),*/ lazy(false) { CND_PRECONDITION(Name != NULL, "Name cannot be NULL"); CND_PRECONDITION(reader != NULL, "reader cannot be NULL"); - _internal->_name = CLStringIntern::intern( Name ); + _name = CLStringIntern::intern( Name ); //_internal->_stringValue = NULL; //_internal->_readerValue = reader; //_internal->_streamValue = NULL; fieldsData = reader; valueType = VALUE_READER; - _internal->boost=1.0f; + boost=1.0f; setConfig(config); } -Field::Field(const TCHAR* Name, const TCHAR* Value, int _config): - _internal(new Internal), lazy(false) +Field::Field(const TCHAR* Name, const TCHAR* Value, int _config, bool duplicateValue): + /*_internal(new Internal),*/ lazy(false) { CND_PRECONDITION(Name != NULL, "Name cannot be NULL"); CND_PRECONDITION(Value != NULL, "value cannot be NULL"); @@ -57,14 +58,17 @@ _CLTHROWA(CL_ERR_IllegalArgument,"cannot store term vector information for a field that is not indexed"); */ - _internal->_name = CLStringIntern::intern( Name ); + _name = CLStringIntern::intern( Name ); //_internal->_stringValue = stringDuplicate( Value ); //_internal->_readerValue = NULL; //_internal->_streamValue = NULL; - fieldsData = stringDuplicate( Value ); + if (duplicateValue) + fieldsData = stringDuplicate( Value ); + else + fieldsData = (void*)Value; valueType = VALUE_STRING; - _internal->boost=1.0f; + boost=1.0f; //config = INDEX_TOKENIZED; // default Field is tokenized and indexed @@ -72,35 +76,35 @@ } Field::Field(const TCHAR* Name, jstreams::StreamBase<char>* Value, int config): - _internal(new Internal), lazy(false) + /*_internal(new Internal),*/ lazy(false) { CND_PRECONDITION(Name != NULL, "Name cannot be NULL"); CND_PRECONDITION(Value != NULL, "value cannot be NULL"); - _internal->_name = CLStringIntern::intern( Name ); + _name = CLStringIntern::intern( Name ); //_internal->_stringValue = NULL; //_internal->_readerValue = NULL; //_internal->_streamValue = Value; fieldsData = Value; valueType = VALUE_STREAM; - _internal->boost=1.0f; + boost=1.0f; setConfig(config); } Field::Field(const TCHAR* Name, int config): - _internal(new Internal), lazy(false) + /*_internal(new Internal),*/ lazy(false) { CND_PRECONDITION(Name != NULL, "Name cannot be NULL"); - _internal->_name = CLStringIntern::intern( Name ); + _name = CLStringIntern::intern( Name ); fieldsData = NULL; valueType = VALUE_NONE; - _internal->boost=1.0f; + boost=1.0f; - setConfig(config); + if (config) setConfig(config); } Field::~Field(){ @@ -108,31 +112,31 @@ //Pre - true //Post - Instance has been destroyed - CLStringIntern::unintern(_internal->_name); + CLStringIntern::unintern(_name); _resetValue(); - delete _internal; + //delete _internal; } /*===============FIELDS=======================*/ -const TCHAR* Field::name() const { return _internal->_name; } ///<returns reference +const TCHAR* Field::name() const { return _name; } ///<returns reference TCHAR* Field::stringValue() const { return (valueType & VALUE_STRING) ? static_cast<TCHAR*>(fieldsData) : NULL; } ///<returns reference Reader* Field::readerValue() const { return (valueType & VALUE_READER) ? static_cast<Reader*>(fieldsData) : NULL; } ///<returns reference jstreams::StreamBase<char>* Field::streamValue() const { return (valueType & VALUE_STREAM) ? static_cast<jstreams::StreamBase<char>*>(fieldsData) : NULL; } ///<returns reference CL_NS(analysis)::TokenStream* Field::tokenStreamValue() const { return (valueType & VALUE_TOKENSTREAM) ? static_cast<CL_NS(analysis)::TokenStream*>(fieldsData) : NULL; } -bool Field::isStored() const { return (_internal->config & STORE_YES) != 0; } -bool Field::isIndexed() const { return (_internal->config & INDEX_TOKENIZED)!=0 || (_internal->config & INDEX_UNTOKENIZED)!=0; } -bool Field::isTokenized() const { return (_internal->config & INDEX_TOKENIZED) != 0; } -bool Field::isCompressed() const { return (_internal->config & STORE_COMPRESS) != 0; } +bool Field::isStored() const { return (config & STORE_YES) != 0; } +bool Field::isIndexed() const { return (config & INDEX_TOKENIZED)!=0 || (config & INDEX_UNTOKENIZED)!=0; } +bool Field::isTokenized() const { return (config & INDEX_TOKENIZED) != 0; } +bool Field::isCompressed() const { return (config & STORE_COMPRESS) != 0; } bool Field::isBinary() const { return (valueType & VALUE_STREAM) && fieldsData!=NULL; } -bool Field::isTermVectorStored() const { return (_internal->config & TERMVECTOR_YES) != 0; } -bool Field::isStoreOffsetWithTermVector() const { return (_internal->config & TERMVECTOR_YES) != 0 && (_internal->config & TERMVECTOR_WITH_OFFSETS) != 0; } -bool Field::isStorePositionWithTermVector() const{ return (_internal->config & TERMVECTOR_YES) != 0 && (_internal->config & TERMVECTOR_WITH_POSITIONS) != 0; } +bool Field::isTermVectorStored() const { return (config & TERMVECTOR_YES) != 0; } +bool Field::isStoreOffsetWithTermVector() const { return (config & TERMVECTOR_YES) != 0 && (config & TERMVECTOR_WITH_OFFSETS) != 0; } +bool Field::isStorePositionWithTermVector() const{ return (config & TERMVECTOR_YES) != 0 && (config & TERMVECTOR_WITH_POSITIONS) != 0; } -bool Field::getOmitNorms() const { return (_internal->config & INDEX_NONORMS) != 0; } -void Field::setOmitNorms(const bool omitNorms) { _internal->config |= INDEX_NONORMS; } +bool Field::getOmitNorms() const { return (config & INDEX_NONORMS) != 0; } +void Field::setOmitNorms(const bool omitNorms) { config |= INDEX_NONORMS; } bool Field::isLazy() const { return lazy; } @@ -160,8 +164,8 @@ //valueType = VALUE_TOKENSTREAM; } -void Field::setBoost(const float_t boost) { this->_internal->boost = boost; } -float_t Field::getBoost() const { return _internal->boost; } +void Field::setBoost(const float_t boost) { this->boost = boost; } +float_t Field::getBoost() const { return boost; } void Field::setConfig(const uint32_t x){ uint32_t newConfig=0; @@ -228,7 +232,7 @@ }else newConfig |= TERMVECTOR_NO; - _internal->config = newConfig; + config = newConfig; } TCHAR* Field::toString() { Modified: branches/lucene2_3_2/src/core/CLucene/document/Field.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/document/Field.h 2008-08-27 12:53:20 UTC (rev 2879) +++ branches/lucene2_3_2/src/core/CLucene/document/Field.h 2008-08-28 13:22:06 UTC (rev 2880) @@ -138,9 +138,21 @@ VALUE_TOKENSTREAM = 8 }; - Field(const TCHAR* name, const TCHAR* value, int _config); + /** + * TCHAR value constructor of Field. Set duplicateValue to false to save on memory allocations when possible + */ + Field(const TCHAR* name, const TCHAR* value, int _config, bool duplicateValue = true); + + /** + * Reader* constructor of Field. + */ Field(const TCHAR* name, CL_NS(util)::Reader* reader, int _config); + + /** + * Stream constructor of Field. + */ Field(const TCHAR* name, jstreams::StreamBase<char>* stream, int _config); + Field(const TCHAR* name, int _config); ///<No value, for lazy loading support ~Field(); @@ -296,12 +308,16 @@ //Set configs using XOR. This resets all the settings //For example, to use term vectors with positions and offsets do: //object->setConfig(TERMVECTOR_WITH_POSITIONS | TERMVECTOR_WITH_OFFSETS); - inline void setConfig(const uint32_t termVector); + inline void setConfig(const uint32_t _config); inline void _resetValue(); void* fieldsData; ValueType valueType; + + const TCHAR* _name; + uint32_t config; + float_t boost; }; CL_NS_END #endif Modified: branches/lucene2_3_2/src/core/CLucene/index/FieldsReader.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/index/FieldsReader.cpp 2008-08-27 12:53:20 UTC (rev 2879) +++ branches/lucene2_3_2/src/core/CLucene/index/FieldsReader.cpp 2008-08-28 13:22:06 UTC (rev 2880) @@ -10,6 +10,7 @@ ////#include "CLucene/util/VoidMap.h" #include "CLucene/util/Misc.h" #include "CLucene/util/subinputstream.h" +#include "CLucene/util/_StringIntern.h" #include "CLucene/store/Directory.h" #include "CLucene/store/IndexInput.h" #include "CLucene/document/Document.h" @@ -34,6 +35,7 @@ int32_t read(const char*& start, int32_t _min, int32_t _max); int64_t skip(int64_t ntoskip); int64_t reset(int64_t pos); + jstreams::SubInputStream<char>* getStream() const; }; FieldsReader::FieldsReader(Directory* d, const char* segment, FieldInfos* fn, int32_t _readBufferSize, int32_t _docStoreOffset, int32_t size): @@ -51,14 +53,13 @@ try { const char* buf = Misc::segmentname(segment,".fdt"); - cloneableFieldsStream = NULL; - fieldsStream = d->openInput( buf, _readBufferSize ); - //fieldsStream = cloneableFieldsStream->clone(); + cloneableFieldsStream = d->openInput( buf, _readBufferSize ); + fieldsStream = cloneableFieldsStream->clone(); _CLDELETE_LCaARRAY( buf ); buf = Misc::segmentname(segment,".fdx"); indexStream = d->openInput( buf, _readBufferSize ); - _CLDELETE_CaARRAY( buf ); + _CLDELETE_LCaARRAY( buf ); if (_docStoreOffset != -1) { // We read only a slice out of this shared fields file @@ -141,7 +142,7 @@ int32_t numFields = fieldsStream->readVInt(); for (int32_t i = 0; i < numFields; i++) { - int32_t fieldNumber = fieldsStream->readVInt(); + const int32_t fieldNumber = fieldsStream->readVInt(); FieldInfo* fi = fieldInfos->fieldInfo(fieldNumber); if ( fi == NULL ) _CLTHROWA(CL_ERR_IO, "Field stream is invalid"); @@ -150,7 +151,38 @@ uint8_t bits = fieldsStream->readByte(); CND_CONDITION(bits <= FieldsWriter::FIELD_IS_COMPRESSED + FieldsWriter::FIELD_IS_TOKENIZED + FieldsWriter::FIELD_IS_BINARY, "invalid field bits"); + + const bool compressed = (bits & FieldsWriter::FIELD_IS_COMPRESSED) != 0; + const bool tokenize = (bits & FieldsWriter::FIELD_IS_TOKENIZED) != 0; + const bool binary = (bits & FieldsWriter::FIELD_IS_BINARY) != 0; + + //TODO: Find an alternative approach here if this list continues to grow beyond the + //list of 5 or 6 currently here. See Lucene 762 for discussion + if (acceptField = LOAD) { + addField(doc, fi, binary, compressed, tokenize); + } + else if (acceptField = LOAD_FOR_MERGE) { + addFieldForMerge(doc, fi, binary, compressed, tokenize); + } + else if (acceptField = LOAD_AND_BREAK){ + addField(doc, fi, binary, compressed, tokenize); + break;//Get out of this loop + } + else if (acceptField = LAZY_LOAD) { + addFieldLazy(doc, fi, binary, compressed, tokenize); + } + else if (acceptField = SIZE){ + skipField(binary, compressed, addFieldSize(doc, fi, binary, compressed)); + } + else if (acceptField = SIZE_AND_BREAK){ + addFieldSize(doc, fi, binary, compressed); + break; + } + else { + skipField(binary, compressed); + } + /* if ((bits & FieldsWriter::FIELD_IS_BINARY) != 0) { int32_t fieldLen = fieldsStream->readVInt(); FieldsReader::FieldsStreamHolder* subStream = new FieldsReader::FieldsStreamHolder(fieldsStream, fieldLen); @@ -205,7 +237,7 @@ fi->name, // name subStream, // read value bits); - + f->setOmitNorms(fi->omitNorms); doc->add(*f); @@ -220,13 +252,14 @@ Field* f = _CLNEW Field( fi->name, // name fvalue, // read value - bits); - _CLDELETE_CARRAY(fvalue); //todo: could optimise this + bits, false); + //_CLDELETE_LCARRAY(fvalue); //todo: could optimise this f->setOmitNorms(fi->omitNorms); doc->add(*f); } } + */ } return true; } @@ -267,7 +300,7 @@ } } -void FieldsReader::addFieldLazy(CL_NS(document)::Document* doc, FieldInfo* fi, const bool binary, +void FieldsReader::addFieldLazy(CL_NS(document)::Document* doc, const FieldInfo* fi, const bool binary, const bool compressed, const bool tokenize) { if (binary) { int32_t toRead = fieldsStream->readVInt(); @@ -282,10 +315,6 @@ //Need to move the pointer ahead by toRead positions fieldsStream->seek(pointer + toRead); } else { - //Field.Store store = Field.Store.YES; - //Field.Index index = getIndexType(fi, tokenize); - //Field.TermVector termVector = getTermVectorType(fi); - LazyField* f = NULL; if (compressed) { int32_t toRead = fieldsStream->readVInt(); @@ -306,21 +335,86 @@ } } -/* -// Add the size of field as a byte[] containing the 4 bytes of the integer byte size (high order byte first; char = 2 bytes) -// Read just the size -- caller must skip the field content to continue reading fields -// Return the size in bytes or chars, depending on field type -int32_t FieldsReader::addFieldSize(const CL_NS(document)::Document* doc, const FieldInfo* fi, const bool binary, const bool compressed) { +// in merge mode we don't uncompress the data of a compressed field +void FieldsReader::addFieldForMerge(CL_NS(document)::Document* doc, const FieldInfo* fi, const bool binary, const bool compressed, const bool tokenize) { + void* data; + Field::ValueType v; + + if ( binary || compressed) { + int32_t toRead = fieldsStream->readVInt(); + FieldsReader::FieldsStreamHolder* subStream = new FieldsReader::FieldsStreamHolder(fieldsStream, toRead); + //final byte[] b = new byte[toRead]; + //fieldsStream->readBytes(b, toRead); + data = subStream->getStream(); + v = Field::VALUE_STREAM; + } else { + data = fieldsStream->readString(); + v = Field::VALUE_STRING; + } + + doc->add(*_CLNEW FieldForMerge(data, v, fi, binary, compressed, tokenize)); +} + +void FieldsReader::addField(CL_NS(document)::Document* doc, const FieldInfo* fi, const bool binary, const bool compressed, const bool tokenize) { + + //we have a binary stored field, and it may be compressed + if (binary) { + const int32_t toRead = fieldsStream->readVInt(); + FieldsReader::FieldsStreamHolder* subStream = new FieldsReader::FieldsStreamHolder(fieldsStream, toRead); + //final byte[] b = new byte[toRead]; + //fieldsStream->readBytes(b, 0, b.length); + if (compressed) { + // we still do not support compressed fields + doc->add(* _CLNEW Field(fi->name, subStream->getStream(), Field::STORE_COMPRESS)); // todo: uncompress(subStream->getStream()) + } + else + doc->add(* _CLNEW Field(fi->name, subStream->getStream(), Field::STORE_YES)); + + } else { + uint8_t bits = 0; + bits |= getIndexType(fi, tokenize); + bits |= getTermVectorType(fi); + + Field* f = NULL; + if (compressed) { + bits |= Field::STORE_COMPRESS; + const int32_t toRead = fieldsStream->readVInt(); + + FieldsStreamHolder* subStream = new FieldsStreamHolder(fieldsStream, toRead); + //final byte[] b = new byte[toRead]; + //fieldsStream.readBytes(b, 0, b.length); + + //todo: we dont have gzip inputstream available, must alert user + //to somehow use a gzip inputstream + f = _CLNEW Field(fi->name, // field name + //todo: new String(uncompress(subStream->getStream()), "UTF-8"), // uncompress the value and add as string + subStream->getStream(), bits); + f->setOmitNorms(fi->omitNorms); + } else { + bits |= Field::STORE_YES; + f = _CLNEW Field(fi->name, // name + fieldsStream->readString(), // read value + bits, false); + f->setOmitNorms(fi->omitNorms); + } + doc->add(*f); + } +} + +int32_t FieldsReader::addFieldSize(CL_NS(document)::Document* doc, const FieldInfo* fi, const bool binary, const bool compressed) { const int32_t size = fieldsStream->readVInt(); const int32_t bytesize = binary || compressed ? size : 2*size; - uint8_t* sizebytes = _CL_NEWARRAY(byte, 4); + /* + uint8_t* sizebytes = _CL_NEWARRAY(uint8_t, 4); sizebytes[0] = (byte) (bytesize>>>24); sizebytes[1] = (byte) (bytesize>>>16); sizebytes[2] = (byte) (bytesize>>> 8); sizebytes[3] = (byte) bytesize ; - doc->add(*_CLNEW Field(fi->name, sizebytes, Field::STORE_YES)); + //todo: doc->add(*_CLNEW Field(fi->name, sizebytes, Field::STORE_YES)); + _CLDELETE_ARRAY(sizebytes); // todo: remove this once doc is being used + */ return size; -}*/ +} CL_NS(document)::Field::TermVector FieldsReader::getTermVectorType(const FieldInfo* fi) { if (fi->storeTermVector) { @@ -387,6 +481,7 @@ this->status = subStream->getStatus(); return ret; } +jstreams::SubInputStream<char>* FieldsReader::FieldsStreamHolder::getStream() const { return subStream; } FieldsReader::LazyField::LazyField(FieldsReader* _parent, const TCHAR* _name, @@ -407,31 +502,32 @@ return localFieldsStream; } -uint8_t* FieldsReader::LazyField::binaryValue() { +jstreams::StreamBase<char>* FieldsReader::LazyField::streamValue() { parent->ensureOpen(); if (fieldsData == NULL) { - uint8_t* b = _CL_NEWARRAY(uint8_t, toRead); + //uint8_t* b = _CL_NEWARRAY(uint8_t, toRead); CL_NS(store)::IndexInput* localFieldsStream = getFieldStream(); localFieldsStream->seek(pointer); - localFieldsStream->readBytes(b, toRead); + FieldsReader::FieldsStreamHolder* subStream = new FieldsReader::FieldsStreamHolder(localFieldsStream, toRead); + //localFieldsStream->readBytes(b, toRead); if (isCompressed()) { //fieldsData = uncompress(b); } else { - fieldsData = b; + fieldsData = subStream->getStream(); } valueType = VALUE_STREAM; } - return static_cast<uint8_t*>(fieldsData); // instanceof byte[] ? (byte[]) fieldsData : null; + return static_cast<jstreams::StreamBase<char>*>(fieldsData); } -CL_NS(util)::Reader* FieldsReader::LazyField::readerValue() { +CL_NS(util)::Reader* FieldsReader::LazyField::readerValue() const { parent->ensureOpen(); return (valueType & VALUE_READER) ? static_cast<CL_NS(util)::Reader*>(fieldsData) : NULL; } -CL_NS(analysis)::TokenStream* FieldsReader::LazyField::tokenStreamValue() { +CL_NS(analysis)::TokenStream* FieldsReader::LazyField::tokenStreamValue() const { parent->ensureOpen(); return (valueType & VALUE_TOKENSTREAM) ? static_cast<CL_NS(analysis)::TokenStream*>(fieldsData) : NULL; } @@ -482,4 +578,46 @@ this->toRead = _toRead; } +const TCHAR* FieldsReader::FieldForMerge::stringValue() const { + return (valueType & VALUE_STRING) ? static_cast<TCHAR*>(fieldsData) : NULL; +} + +CL_NS(util)::Reader* FieldsReader::FieldForMerge::readerValue() const { + // not needed for merge + return NULL; +} + +jstreams::StreamBase<char>* FieldsReader::FieldForMerge::streamValue() const { + return (valueType & VALUE_STREAM) ? static_cast<jstreams::StreamBase<char>*>(fieldsData) : NULL; +} + +CL_NS(analysis)::TokenStream* FieldsReader::FieldForMerge::tokenStreamValue() const { + // not needed for merge + return NULL; +} + +FieldsReader::FieldForMerge::FieldForMerge(void* _value, ValueType _type, const FieldInfo* fi, const bool binary, const bool compressed, const bool tokenize) : Field(fi->name, 0) { + + uint32_t bits = STORE_YES; + if (compressed) bits |= STORE_COMPRESS; + + this->fieldsData = _value; + this->valueType = _type; + + //this->isTokenized = tokenize; + if (tokenize) bits |= INDEX_TOKENIZED; + + if (fi->isIndexed && !tokenize) bits |= INDEX_UNTOKENIZED; + //this->isIndexed = fi->isIndexed; + + if (fi->omitNorms) bits |= INDEX_NONORMS; + //this->omitNorms = fi->omitNorms; + + if (fi->storeOffsetWithTermVector) bits |= TERMVECTOR_WITH_OFFSETS; + if (fi->storePositionWithTermVector) bits |= TERMVECTOR_WITH_POSITIONS; + if (fi->storeTermVector) bits |= TERMVECTOR_YES; + + setConfig(bits); +} + CL_NS_END Modified: branches/lucene2_3_2/src/core/CLucene/index/_FieldsReader.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/index/_FieldsReader.h 2008-08-27 12:53:20 UTC (rev 2879) +++ branches/lucene2_3_2/src/core/CLucene/index/_FieldsReader.h 2008-08-28 13:22:06 UTC (rev 2880) @@ -88,8 +88,19 @@ void skipField(const bool binary, const bool compressed); void skipField(const bool binary, const bool compressed, const int32_t toRead); - void addFieldLazy(CL_NS(document)::Document* doc, FieldInfo* fi, const bool binary, const bool compressed, const bool tokenize); + void addFieldLazy(CL_NS(document)::Document* doc, const FieldInfo* fi, const bool binary, const bool compressed, const bool tokenize); + /** Add the size of field as a byte[] containing the 4 bytes of the integer byte size (high order byte first; char = 2 bytes) + * Read just the size -- caller must skip the field content to continue reading fields + * Return the size in bytes or chars, depending on field type + */ + int32_t addFieldSize(CL_NS(document)::Document* doc, const FieldInfo* fi, const bool binary, const bool compressed); + + // in merge mode we don't uncompress the data of a compressed field + void addFieldForMerge(CL_NS(document)::Document* doc, const FieldInfo* fi, const bool binary, const bool compressed, const bool tokenize); + + void addField(CL_NS(document)::Document* doc, const FieldInfo* fi, const bool binary, const bool compressed, const bool tokenize); + CL_NS(document)::Field::TermVector getTermVectorType(const FieldInfo* fi); CL_NS(document)::Field::Index getIndexType(const FieldInfo* fi, const bool tokenize); @@ -114,12 +125,12 @@ /** The value of the field in Binary, or null. If null, the Reader value, * String value, or TokenStream value is used. Exactly one of stringValue(), * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ - uint8_t* binaryValue(); + jstreams::StreamBase<char>* streamValue(); /** The value of the field as a Reader, or null. If null, the String value, * binary value, or TokenStream value is used. Exactly one of stringValue(), * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ - CL_NS(util)::Reader* readerValue(); + CL_NS(util)::Reader* readerValue() const; /** The value of the field as a String, or null. If null, the Reader value, * binary value, or TokenStream value is used. Exactly one of stringValue(), @@ -129,7 +140,7 @@ /** The value of the field as a TokesStream, or null. If null, the Reader value, * String value, or binary value is used. Exactly one of stringValue(), * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ - CL_NS(analysis)::TokenStream* tokenStreamValue(); + CL_NS(analysis)::TokenStream* tokenStreamValue() const; int64_t getPointer() const; void setPointer(const int64_t _pointer); @@ -137,6 +148,18 @@ int32_t getToRead() const; void setToRead(const int32_t _toRead); }; + + // Instances of this class hold field properties and data + // for merge + static class FieldForMerge : public CL_NS(document)::Field { + public: + const TCHAR* stringValue() const; + CL_NS(util)::Reader* readerValue() const; + jstreams::StreamBase<char>* streamValue() const; + CL_NS(analysis)::TokenStream* tokenStreamValue() const; + + FieldForMerge(void* _value, ValueType _type, const FieldInfo* fi, const bool binary, const bool compressed, const bool tokenize); + }; }; CL_NS_END #endif Modified: branches/lucene2_3_2/src/test/search/TestSearch.cpp =================================================================== --- branches/lucene2_3_2/src/test/search/TestSearch.cpp 2008-08-27 12:53:20 UTC (rev 2879) +++ branches/lucene2_3_2/src/test/search/TestSearch.cpp 2008-08-28 13:22:06 UTC (rev 2880) @@ -216,12 +216,16 @@ _TestSearchesRun(&a,s, _T(";*") ); } -void SearchTest(CuTest *tc) { +void SearchTest(CuTest *tc, bool bram) { uint64_t start = Misc::currentTimeMillis(); SimpleAnalyzer analyzer; - RAMDirectory ram; - IndexWriter writer( &ram, &analyzer, true); + + char fsdir[CL_MAX_PATH]; + sprintf(fsdir,"%s/%s",cl_tempDir, "test.search"); + Directory* ram = (bram?(Directory*)_CLNEW RAMDirectory():(Directory*)FSDirectory::getDirectory(fsdir, true) ); + + IndexWriter writer( ram, &analyzer, true); writer.setUseCompoundFile(false); const TCHAR* docs[] = { _T("a b c d e"), @@ -243,7 +247,13 @@ } writer.close(); - IndexReader* reader = IndexReader::open(&ram); + if (!bram){ + ram->close(); + _CLDECDELETE(ram); + ram = (Directory*)FSDirectory::getDirectory(fsdir, false); + } + + IndexReader* reader = IndexReader::open(ram); IndexSearcher searcher(reader); const TCHAR* queries[] = { _T("\"a b\""), @@ -272,6 +282,9 @@ reader->close(); _CLDELETE( reader ); + ram->close(); + _CLDECDELETE(ram); + CuMessageA (tc,"took %d milliseconds\n", Misc::currentTimeMillis()-start); } @@ -346,10 +359,14 @@ searcher.close(); } +void ramSearchTest(CuTest *tc) { SearchTest(tc, true); } +void fsSearchTest(CuTest *tc) { SearchTest(tc, false); } + CuSuite *testsearch(void) { CuSuite *suite = CuSuiteNew(_T("CLucene Search Test")); - SUITE_ADD_TEST(suite, SearchTest); + SUITE_ADD_TEST(suite, ramSearchTest); + SUITE_ADD_TEST(suite, fsSearchTest); SUITE_ADD_TEST(suite, testNormEncoding); SUITE_ADD_TEST(suite, testSrchManyHits); This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-09-27 09:34:25
|
Revision: 2897 http://clucene.svn.sourceforge.net/clucene/?rev=2897&view=rev Author: ustramooner Date: 2008-09-27 09:34:20 +0000 (Sat, 27 Sep 2008) Log Message: ----------- threading test and a test suite for it. macros for creating threads also provided Modified Paths: -------------- branches/lucene2_3_2/src/shared/CLucene/LuceneThreads.h branches/lucene2_3_2/src/shared/CLucene/config/_threads.h branches/lucene2_3_2/src/shared/CLucene/config/threads.cpp branches/lucene2_3_2/src/test/index/TestReuters.cpp Modified: branches/lucene2_3_2/src/shared/CLucene/LuceneThreads.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/LuceneThreads.h 2008-09-27 09:31:56 UTC (rev 2896) +++ branches/lucene2_3_2/src/shared/CLucene/LuceneThreads.h 2008-09-27 09:34:20 UTC (rev 2897) @@ -18,14 +18,19 @@ #define STATIC_DEFINE_MUTEX(x) #define _LUCENE_CURRTHREADID 1 #define _LUCENE_THREADID_TYPE char + #define _LUCENE_THREAD_FUNC(name, argName) int name(void* argName) + #define _LUCENE_THREAD_CREATE(value, func, arg) func(arg) + #define _LUCENE_THREAD_JOIN(value) //nothing to do... #else #if defined(_LUCENE_DONTIMPLEMENT_THREADMUTEX) //do nothing #else - #if defined(_CL_HAVE_PTHREAD) + #if defined(_CL_HAVE_PTHREAD) #define _LUCENE_THREADID_TYPE pthread_t - class CLUCENE_SHARED_EXPORT mutex_thread + #define _LUCENE_THREAD_FUNC(name, argName) void* name(void* argName) //< use this macro to correctly define the thread start routine + typedef void* (luceneThreadStartRoutine)(void* lpThreadParameter ); + class CLUCENE_SHARED_EXPORT mutex_thread { private: struct Internal; @@ -37,11 +42,15 @@ void lock(); void unlock(); static _LUCENE_THREADID_TYPE _GetCurrentThreadId(); + static _LUCENE_THREADID_TYPE CreateThread(luceneThreadStartRoutine* func, void* arg); + static void JoinThread(_LUCENE_THREADID_TYPE id); }; #elif defined(_CL_HAVE_WIN32_THREADS) #define _LUCENE_THREADID_TYPE uint64_t - class CLUCENE_SHARED_EXPORT mutex_thread + #define _LUCENE_THREAD_FUNC(name, argName) void __stdcall name(void* argName) //< use this macro to correctly define the thread start routine + typedef void (__stdcall luceneThreadStartRoutine)(void* lpThreadParameter ); + class CLUCENE_SHARED_EXPORT mutex_thread { private: struct Internal; @@ -53,14 +62,17 @@ void lock(); void unlock(); static _LUCENE_THREADID_TYPE _GetCurrentThreadId(); + static _LUCENE_THREADID_TYPE CreateThread(luceneThreadStartRoutine* func, void* arg); + static void JoinThread(_LUCENE_THREADID_TYPE id); }; - #else #error A valid thread library was not found #endif //mutex types + #define _LUCENE_THREAD_CREATE(func, arg) mutex_thread::CreateThread(func,arg) + #define _LUCENE_THREAD_JOIN(id) mutex_thread::JoinThread(id) #define _LUCENE_CURRTHREADID mutex_thread::_GetCurrentThreadId() #define _LUCENE_THREADMUTEX CL_NS(util)::mutex_thread #endif //don't implement Modified: branches/lucene2_3_2/src/shared/CLucene/config/_threads.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/config/_threads.h 2008-09-27 09:31:56 UTC (rev 2896) +++ branches/lucene2_3_2/src/shared/CLucene/config/_threads.h 2008-09-27 09:34:20 UTC (rev 2897) @@ -25,11 +25,24 @@ void * LockSemaphore; _cl_dword_t SpinCount; }; + + __declspec(dllimport) void __stdcall InitializeCriticalSection(CRITICAL_SECTION *); __declspec(dllimport) void __stdcall EnterCriticalSection(CRITICAL_SECTION *); __declspec(dllimport) void __stdcall LeaveCriticalSection(CRITICAL_SECTION *); __declspec(dllimport) void __stdcall DeleteCriticalSection(CRITICAL_SECTION *); __declspec(dllimport) unsigned long __stdcall GetCurrentThreadId(); + + typedef struct _SECURITY_ATTRIBUTES + { + _cl_dword_t nLength; + void* lpSecurityDescriptor; + bool bInheritHandle; + } SECURITY_ATTRIBUTES; + __declspec(dllimport) _cl_dword_t __stdcall WaitForSingleObject( void* hHandle, _cl_dword_t dwMilliseconds ); + void* _beginthread( void( __stdcall *start_address )( void * ), unsigned stack_size, void *arglist ); + + } #endif //_WINBASE_ #elif defined(_CL_HAVE_PTHREAD) @@ -72,6 +85,7 @@ }; // min_buckets = 2 ^^ N, 0 < N bool operator()( pthread_t t1, pthread_t t2 ) const{ + //pthread_equal should be used, but it returns only non-zero if equal, so we can't use it for order compare return t1 < t2; } }; Modified: branches/lucene2_3_2/src/shared/CLucene/config/threads.cpp =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/config/threads.cpp 2008-09-27 09:31:56 UTC (rev 2896) +++ branches/lucene2_3_2/src/shared/CLucene/config/threads.cpp 2008-09-27 09:34:20 UTC (rev 2897) @@ -7,6 +7,7 @@ #include "CLucene/_SharedHeader.h" #include "CLucene/LuceneThreads.h" #include "_threads.h" +#include <assert.h> CL_NS_DEF(util) @@ -57,9 +58,20 @@ _LUCENE_THREADID_TYPE mutex_thread::_GetCurrentThreadId(){ return GetCurrentThreadId(); } + + _LUCENE_THREADID_TYPE mutex_thread::CreateThread(luceneThreadStartRoutine* func, void* arg){ + return (_LUCENE_THREADID_TYPE) ::_beginthread (func, 0, arg); + } + void mutex_thread::JoinThread(_LUCENE_THREADID_TYPE id){ + WaitForSingleObject((void*)id, 0xFFFFFFFF); + } #elif defined(_CL_HAVE_PTHREAD) + #ifndef _REENTRANT + #error ACK! You need to compile with _REENTRANT defined since this uses threads + #endif + #ifdef _CL_HAVE_PTHREAD_MUTEX_RECURSIVE bool mutex_pthread_attr_initd=false; pthread_mutexattr_t mutex_thread_attr; @@ -126,6 +138,15 @@ _LUCENE_THREADID_TYPE mutex_thread::_GetCurrentThreadId(){ return pthread_self(); } + + _LUCENE_THREADID_TYPE mutex_thread::CreateThread(luceneThreadStartRoutine* func, void* arg){ + _LUCENE_THREADID_TYPE ret; + assert(pthread_create(&ret, NULL, func, arg) == 0 ); + return ret; + } + void mutex_thread::JoinThread(_LUCENE_THREADID_TYPE id){ + pthread_join(id, NULL); + } void mutex_thread::lock() { Modified: branches/lucene2_3_2/src/test/index/TestReuters.cpp =================================================================== --- branches/lucene2_3_2/src/test/index/TestReuters.cpp 2008-09-27 09:31:56 UTC (rev 2896) +++ branches/lucene2_3_2/src/test/index/TestReuters.cpp 2008-09-27 09:34:20 UTC (rev 2897) @@ -1,6 +1,7 @@ #include "test.h" #include "CLucene/util/dirent.h" #include "CLucene/util/Reader.h" +#include "CLucene/LuceneThreads.h" #ifdef _CL_HAVE_SYS_STAT_H #include <sys/stat.h> @@ -267,7 +268,68 @@ _CLDELETE(reader1); _CLDELETE(reader2); } + + #define threadsCount 10 + + StandardAnalyzer threadAnalyzer; + void threadSearch(IndexSearcher* searcher, const TCHAR* qry){ + Query* q = NULL; + Hits* h = NULL; + try{ + q = QueryParser::parse(qry , _T("contents"), &threadAnalyzer); + if ( q != NULL ){ + h = searcher->search( q ); + + if ( h->length() > 0 ){ + //check for explanation memory leaks... + Explanation expl1; + searcher->explain(q, h->id(0), &expl1); + TCHAR* tmp = expl1.toString(); + _CLDELETE_CARRAY(tmp); + if ( h->length() > 1 ){ //do a second one just in case + Explanation expl2; + searcher->explain(q, h->id(1), &expl2); + tmp = expl2.toString(); + _CLDELETE_CARRAY(tmp); + } + } + } + }_CLFINALLY( + _CLDELETE(h); + _CLDELETE(q); + ); + } + _LUCENE_THREAD_FUNC(threadedSearcherTest, arg){ + IndexSearcher* searcher = (IndexSearcher*)arg; + printf("thread started :-)...\n"); + + for ( int i=0;i<100;i++ ){ + threadSearch(searcher, _T("test") ); + threadSearch(searcher, _T("reuters") ); + threadSearch(searcher, _T("data") ); + } + printf ("done...\n"); + } + + void testThreaded(CuTest* tc){ + CLUCENE_ASSERT(reuters_ready); + IndexSearcher searcher(reuters_origdirectory); + + //read using multiple threads... + _LUCENE_THREADID_TYPE threads[threadsCount]; + + int i; + for ( i=0;i<threadsCount;i++ ) + threads[i] = _LUCENE_THREAD_CREATE(&threadedSearcherTest, &searcher); + + CL_NS(util)::Misc::sleep(3000); + for ( i=0;i<threadsCount;i++ ) + _LUCENE_THREAD_JOIN(threads[i]); + + searcher.close(); + } + CuSuite *testreuters(void) { CuSuite *suite = CuSuiteNew(_T("CLucene Reuters Test")); @@ -279,6 +341,10 @@ SUITE_ADD_TEST(suite, testReuters); //SUITE_ADD_TEST(suite, testByteForByte); this test rarely works currently, use more robust by section test... SUITE_ADD_TEST(suite, testBySection); + + //we still do this, but it'll be slow because the 'threads' will be run serially. + + SUITE_ADD_TEST(suite, testThreaded); return suite; } // EOF This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-09-27 09:47:39
|
Revision: 2899 http://clucene.svn.sourceforge.net/clucene/?rev=2899&view=rev Author: ustramooner Date: 2008-09-27 09:47:28 +0000 (Sat, 27 Sep 2008) Log Message: ----------- moved CLuceneError out of shared and into core. makes it easier for linking, since you only need to link to core. Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/CLMonolithic.cpp branches/lucene2_3_2/src/core/CLucene/StdHeader.h branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp branches/lucene2_3_2/src/shared/CLucene/util/StringBuffer.cpp branches/lucene2_3_2/src/shared/CMakeLists.txt Added Paths: ----------- branches/lucene2_3_2/src/core/CLucene/debug/error.cpp branches/lucene2_3_2/src/core/CLucene/debug/error.h Removed Paths: ------------- branches/lucene2_3_2/src/shared/CLucene/debug/error.cpp branches/lucene2_3_2/src/shared/CLucene/debug/error.h Modified: branches/lucene2_3_2/src/core/CLucene/CLMonolithic.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/CLMonolithic.cpp 2008-09-27 09:36:09 UTC (rev 2898) +++ branches/lucene2_3_2/src/core/CLucene/CLMonolithic.cpp 2008-09-27 09:47:28 UTC (rev 2899) @@ -11,6 +11,7 @@ * note: when creating a project add either this file, or all the other .cpp files, not both! */ #include "CLucene/StdHeader.cpp" +#include "CLucene/debug/error.cpp" #include "CLucene/analysis/Analyzers.cpp" #include "CLucene/analysis/AnalysisHeader.cpp" #include "CLucene/analysis/standard/StandardAnalyzer.cpp" Modified: branches/lucene2_3_2/src/core/CLucene/StdHeader.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/StdHeader.h 2008-09-27 09:36:09 UTC (rev 2898) +++ branches/lucene2_3_2/src/core/CLucene/StdHeader.h 2008-09-27 09:47:28 UTC (rev 2899) @@ -19,6 +19,9 @@ //shared header #include "CLucene/SharedHeader.h" +//error handling macros/functions +#include "CLucene/debug/error.h" + //todo: would be good to deprecate this... it's ugly #define StringArrayWithDeletor CL_NS(util)::CLVector<TCHAR*, CL_NS(util)::Deletor::tcArray > #define StringArray std::vector<TCHAR*> Copied: branches/lucene2_3_2/src/core/CLucene/debug/error.cpp (from rev 2892, branches/lucene2_3_2/src/shared/CLucene/debug/error.cpp) =================================================================== --- branches/lucene2_3_2/src/core/CLucene/debug/error.cpp (rev 0) +++ branches/lucene2_3_2/src/core/CLucene/debug/error.cpp 2008-09-27 09:47:28 UTC (rev 2899) @@ -0,0 +1,74 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/_SharedHeader.h" +#include "error.h" +#include "CLucene/util/Misc.h" +CL_NS_USE(util) + + +#ifdef _LUCENE_DISABLE_EXCEPTIONS + #ifdef _LUCENE_PRAGMA_WARNINGS + #pragma message ("==================Lucene exceptions are disabled==================") + #else + #warning "==================Lucene exceptions are disabled==================" + #endif +#else + CLuceneError::CLuceneError(int num, const char* str, bool ownstr) + { + error_number = num; + _awhat=STRDUP_AtoA(str); + _twhat=NULL; + if ( ownstr ) + _CLDELETE_CaARRAY(str); + } + + CLuceneError::CLuceneError(const CLuceneError& clone) + { + this->error_number = clone.error_number; + this->_awhat = NULL; + this->_twhat = NULL; + + if ( clone._awhat != NULL ) + this->_awhat = STRDUP_AtoA(clone._awhat); + if ( clone._twhat != NULL ) + this->_twhat = STRDUP_TtoT(clone._twhat); + } + CLuceneError::~CLuceneError() throw(){ + _CLDELETE_CARRAY(_twhat); + _CLDELETE_CaARRAY(_awhat); + } + char* CLuceneError::what(){ +#ifdef _ASCII + if ( _twhat != NULL ) + return _twhat; +#endif + if ( _awhat == NULL ) + _awhat = STRDUP_TtoA(_twhat); + return _awhat; + } + TCHAR* CLuceneError::twhat(){ +#ifdef _ASCII + if ( _awhat != NULL ) + return _awhat; +#endif + if ( _twhat == NULL ) + _twhat = STRDUP_AtoT(_awhat); + return _twhat; + } + +#ifndef _ASCII + CLuceneError::CLuceneError(int num, const TCHAR* str, bool ownstr) + { + error_number = num; + _awhat=NULL; + _twhat=STRDUP_TtoT(str); + if ( ownstr ) + _CLDELETE_CARRAY(str); + } +#endif + +#endif //_LUCENE_DISABLE_EXCEPTIONS Property changes on: branches/lucene2_3_2/src/core/CLucene/debug/error.cpp ___________________________________________________________________ Added: svn:mergeinfo + Copied: branches/lucene2_3_2/src/core/CLucene/debug/error.h (from rev 2892, branches/lucene2_3_2/src/shared/CLucene/debug/error.h) =================================================================== --- branches/lucene2_3_2/src/core/CLucene/debug/error.h (rev 0) +++ branches/lucene2_3_2/src/core/CLucene/debug/error.h 2008-09-27 09:47:28 UTC (rev 2899) @@ -0,0 +1,87 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_debug_error_ +#define _lucene_debug_error_ + + +#define CL_ERR_UNKNOWN -1 +#define CL_ERR_IO 1 +#define CL_ERR_NullPointer 2 +#define CL_ERR_Runtime 3 +#define CL_ERR_IllegalArgument 4 +#define CL_ERR_Parse 5 +#define CL_ERR_TokenMgr 6 +#define CL_ERR_UnsupportedOperation 7 +#define CL_ERR_InvalidState 8 +#define CL_ERR_IndexOutOfBounds 9 +#define CL_ERR_TooManyClauses 10 +#define CL_ERR_RAMTransaction 11 +#define CL_ERR_InvalidCast 12 +#define CL_ERR_IllegalState 13 // Sub-error: AlreadyClosed +#define CL_ERR_UnknownOperator 14 +#define CL_ERR_ConcurrentModification 15 +#define CL_ERR_CorruptIndex 16 + + + +//////////////////////////////////////////////////////// +//error try/throw/catch definitions +//////////////////////////////////////////////////////// +#ifdef _CL_DISABLE_NATIVE_EXCEPTIONS + #include <setjmp.h> + + /* + #define try struct pj_exception_state_t pj_x_except__; int pj_x_code__; \ + if(1){ \ + pj_push_exception_handler_(&pj_x_except__); \ + pj_x_code__ = pj_setjmp(pj_x_except__.state); \ + if (pj_x_code__ == 0) + #define _CLCATCHEND pj_pop_exception_handler_(); \ + } else {} + #define _CLCATCH else if (pj_x_code__ == (id)) _CLCATCHEND + #define _CLCATCHANY else _CLCATCHEND + #define _RETHROW pj_throw_exception_(pj_x_code__) + + #define _CLFINALLY(x) else{x _RETHROW}_CLCATCHEND x + #define _CLTHROWA(number) pj_throw_exception_(number) + #define _CLTHROWT(number) pj_throw_exception_(number) + #define _THROWA_DEL(number) _CLDELETE_CaARRAY(str); pj_throw_exception_(number) + #define _THROWT_DEL(number) _CLDELETE_CARRAY(str); pj_throw_exception_(number) + + */ +#else + class CLUCENE_EXPORT CLuceneError + { + int error_number; + char* _awhat; + TCHAR* _twhat; + public: + CLuceneError(const CLuceneError& clone); + CLuceneError(int num, const char* str, bool ownstr); +#ifdef _UCS2 + CLuceneError(int num, const TCHAR* str, bool ownstr); +#endif + int number(){return error_number;} + char* what(); + TCHAR* twhat(); + ~CLuceneError() throw(); + }; + + //#define _THROWS //does nothing + #define _TRY try + #define _CLFINALLY(x) catch(...){ x; throw; } x //note: code x is not run if return is called + #define _CLTHROWA(number, str) throw CLuceneError(number, str,false) + #define _CLTHROWT(number, str) throw CLuceneError(number, str,false) + #define _CLTHROWA_DEL(number, str) throw CLuceneError(number, str,true) //throw a string ensures the value is deleted + #define _CLTHROWT_DEL(number, str) throw CLuceneError(number, str,true) //throw a string ensures the value is deleted + + +#endif //_LUCENE_DISABLE_EXCEPTIONS +// +//////////////////////////////////////////////////////// + +#endif Property changes on: branches/lucene2_3_2/src/core/CLucene/debug/error.h ___________________________________________________________________ Added: svn:mergeinfo + Modified: branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-09-27 09:36:09 UTC (rev 2898) +++ branches/lucene2_3_2/src/shared/CLucene/SharedHeader.h 2008-09-27 09:47:28 UTC (rev 2899) @@ -86,6 +86,9 @@ #define CL_STRUCT_DEF(sub,clazz) namespace lucene { namespace sub{ struct clazz; } } #define CL_CLASS_DEF(sub,clazz) namespace lucene { namespace sub{ class clazz; } } #define CL_CLASS_DEF2(sub,sub2, clazz) namespace lucene { namespace sub{ namespace sub2{ class clazz; } } } + + #define CL_TEMPATE_DEF(sub, clazz, typedefs) namespace lucene { namespace sub{ template<typedefs> class clazz; }} + #define CL_TYPE_DEF(sub, clazz, def) namespace lucene { namespace sub{ typedef def clazz; }} #else #define CL_NS_DEF(sub) #define CL_NS_DEF2(sub, sub2) @@ -190,9 +193,6 @@ #include "CLucene/debug/lucenebase.h" //////////////////////////////////////////////////////// -//error handling macros/functions -#include "CLucene/debug/error.h" - //memory handling macros/functions #include "CLucene/debug/mem.h" Deleted: branches/lucene2_3_2/src/shared/CLucene/debug/error.cpp =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/debug/error.cpp 2008-09-27 09:36:09 UTC (rev 2898) +++ branches/lucene2_3_2/src/shared/CLucene/debug/error.cpp 2008-09-27 09:47:28 UTC (rev 2899) @@ -1,73 +0,0 @@ -/*------------------------------------------------------------------------------ -* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team -* -* Distributable under the terms of either the Apache License (Version 2.0) or -* the GNU Lesser General Public License, as specified in the COPYING file. -------------------------------------------------------------------------------*/ -#include "CLucene/_SharedHeader.h" -#include "CLucene/util/Misc.h" -CL_NS_USE(util) - - -#ifdef _LUCENE_DISABLE_EXCEPTIONS - #ifdef _LUCENE_PRAGMA_WARNINGS - #pragma message ("==================Lucene exceptions are disabled==================") - #else - #warning "==================Lucene exceptions are disabled==================" - #endif -#else - CLuceneError::CLuceneError(int num, const char* str, bool ownstr) - { - error_number = num; - _awhat=STRDUP_AtoA(str); - _twhat=NULL; - if ( ownstr ) - _CLDELETE_CaARRAY(str); - } - - CLuceneError::CLuceneError(const CLuceneError& clone) - { - this->error_number = clone.error_number; - this->_awhat = NULL; - this->_twhat = NULL; - - if ( clone._awhat != NULL ) - this->_awhat = STRDUP_AtoA(clone._awhat); - if ( clone._twhat != NULL ) - this->_twhat = STRDUP_TtoT(clone._twhat); - } - CLuceneError::~CLuceneError() throw(){ - _CLDELETE_CARRAY(_twhat); - _CLDELETE_CaARRAY(_awhat); - } - char* CLuceneError::what(){ -#ifdef _ASCII - if ( _twhat != NULL ) - return _twhat; -#endif - if ( _awhat == NULL ) - _awhat = STRDUP_TtoA(_twhat); - return _awhat; - } - TCHAR* CLuceneError::twhat(){ -#ifdef _ASCII - if ( _awhat != NULL ) - return _awhat; -#endif - if ( _twhat == NULL ) - _twhat = STRDUP_AtoT(_awhat); - return _twhat; - } - -#ifndef _ASCII - CLuceneError::CLuceneError(int num, const TCHAR* str, bool ownstr) - { - error_number = num; - _awhat=NULL; - _twhat=STRDUP_TtoT(str); - if ( ownstr ) - _CLDELETE_CARRAY(str); - } -#endif - -#endif //_LUCENE_DISABLE_EXCEPTIONS Deleted: branches/lucene2_3_2/src/shared/CLucene/debug/error.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/debug/error.h 2008-09-27 09:36:09 UTC (rev 2898) +++ branches/lucene2_3_2/src/shared/CLucene/debug/error.h 2008-09-27 09:47:28 UTC (rev 2899) @@ -1,86 +0,0 @@ -/*------------------------------------------------------------------------------ -* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team -* -* Distributable under the terms of either the Apache License (Version 2.0) or -* the GNU Lesser General Public License, as specified in the COPYING file. -------------------------------------------------------------------------------*/ -#ifndef _lucene_debug_error_ -#define _lucene_debug_error_ - - -#define CL_ERR_UNKNOWN -1 -#define CL_ERR_IO 1 -#define CL_ERR_NullPointer 2 -#define CL_ERR_Runtime 3 -#define CL_ERR_IllegalArgument 4 -#define CL_ERR_Parse 5 -#define CL_ERR_TokenMgr 6 -#define CL_ERR_UnsupportedOperation 7 -#define CL_ERR_InvalidState 8 -#define CL_ERR_IndexOutOfBounds 9 -#define CL_ERR_TooManyClauses 10 -#define CL_ERR_RAMTransaction 11 -#define CL_ERR_InvalidCast 12 -#define CL_ERR_IllegalState 13 // Sub-error: AlreadyClosed -#define CL_ERR_UnknownOperator 14 -#define CL_ERR_ConcurrentModification 15 -#define CL_ERR_CorruptIndex 16 - - - -//////////////////////////////////////////////////////// -//error try/throw/catch definitions -//////////////////////////////////////////////////////// -#ifdef _CL_DISABLE_NATIVE_EXCEPTIONS - #include <setjmp.h> - /* - #define try struct pj_exception_state_t pj_x_except__; int pj_x_code__; \ - if(1){ \ - pj_push_exception_handler_(&pj_x_except__); \ - pj_x_code__ = pj_setjmp(pj_x_except__.state); \ - if (pj_x_code__ == 0) - #define _CLCATCHEND pj_pop_exception_handler_(); \ - } else {} - #define _CLCATCH else if (pj_x_code__ == (id)) _CLCATCHEND - #define _CLCATCHANY else _CLCATCHEND - #define _RETHROW pj_throw_exception_(pj_x_code__) - - #define _CLFINALLY(x) else{x _RETHROW}_CLCATCHEND x - #define _CLTHROWA(number) pj_throw_exception_(number) - #define _CLTHROWT(number) pj_throw_exception_(number) - #define _THROWA_DEL(number) _CLDELETE_CaARRAY(str); pj_throw_exception_(number) - #define _THROWT_DEL(number) _CLDELETE_CARRAY(str); pj_throw_exception_(number) - - */ -#else - class CLUCENE_SHARED_EXPORT CLuceneError - { - int error_number; - char* _awhat; - TCHAR* _twhat; - public: - CLuceneError(const CLuceneError& clone); - CLuceneError(int num, const char* str, bool ownstr); -#ifdef _UCS2 - CLuceneError(int num, const TCHAR* str, bool ownstr); -#endif - int number(){return error_number;} - char* what(); - TCHAR* twhat(); - ~CLuceneError() throw(); - }; - - //#define _THROWS //does nothing - #define _TRY try - #define _CLFINALLY(x) catch(...){ x; throw; } x //note: code x is not run if return is called - #define _CLTHROWA(number, str) throw CLuceneError(number, str,false) - #define _CLTHROWT(number, str) throw CLuceneError(number, str,false) - #define _CLTHROWA_DEL(number, str) throw CLuceneError(number, str,true) //throw a string ensures the value is deleted - #define _CLTHROWT_DEL(number, str) throw CLuceneError(number, str,true) //throw a string ensures the value is deleted - - -#endif //_LUCENE_DISABLE_EXCEPTIONS -// -//////////////////////////////////////////////////////// - -#endif Modified: branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp 2008-09-27 09:36:09 UTC (rev 2898) +++ branches/lucene2_3_2/src/shared/CLucene/util/Misc.cpp 2008-09-27 09:47:28 UTC (rev 2899) @@ -51,13 +51,11 @@ return hashCode; } -//ok, these are the exceptions, but these never -//exist on non-msvc platform, so lets put it here int64_t Misc::filelength(int filehandle) { struct cl_stat_t info; if (fileHandleStat(filehandle, &info) == -1) - _CLTHROWA( CL_ERR_IO,"fileStat error" ); + return -1; return info.st_size; } @@ -125,7 +123,7 @@ #else struct timeval tstruct; if (gettimeofday(&tstruct, NULL) < 0) { - _CLTHROWA(CL_ERR_Runtime,"Error in gettimeofday call."); + return 0; } return (((uint64_t) tstruct.tv_sec) * 1000) + tstruct.tv_usec / 1000; Modified: branches/lucene2_3_2/src/shared/CLucene/util/StringBuffer.cpp =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/util/StringBuffer.cpp 2008-09-27 09:36:09 UTC (rev 2898) +++ branches/lucene2_3_2/src/shared/CLucene/util/StringBuffer.cpp 2008-09-27 09:47:28 UTC (rev 2899) @@ -7,6 +7,7 @@ #include "CLucene/_ApiHeader.h" #include "StringBuffer.h" #include "Misc.h" +#include <assert.h> CL_NS_DEF(util) @@ -157,8 +158,7 @@ //using sprintf("%f" was not reliable on other plaforms... we use a custom float convertor //bvk: also, using sprintf and %f seems excessivelly slow - if(digits>8) - _CLTHROWA(CL_ERR_IllegalArgument,"Too many digits..."); + assert(digits <= 8); //the maximum number of characters that int64 will hold is 23. so we need 23*2+2 TCHAR buf[48]; //the buffer to hold @@ -304,8 +304,7 @@ //More aggressive growth strategy to offset smaller default buffer size: if ( !bufferOwner ){ - if ( bufferLength<minLength ) - _CLTHROWA(CL_ERR_IllegalArgument,"[StringBuffer::grow] un-owned buffer could not be grown"); + assert(bufferLength>=minLength); return; } Modified: branches/lucene2_3_2/src/shared/CMakeLists.txt =================================================================== --- branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-09-27 09:36:09 UTC (rev 2898) +++ branches/lucene2_3_2/src/shared/CMakeLists.txt 2008-09-27 09:47:28 UTC (rev 2899) @@ -266,8 +266,7 @@ SET(${result} ${rel}/CLucene/SharedHeader.cpp - ${rel}/CLucene/debug/error.cpp - ${rel}/CLucene/config/gunichartables.cpp + ${rel}/CLucene/config/gunichartables.cpp ${rel}/CLucene/config/repl_tcslwr.cpp ${rel}/CLucene/config/repl_tcstoll.cpp ${rel}/CLucene/config/repl_tcscasecmp.cpp This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ust...@us...> - 2008-10-02 14:19:30
|
Revision: 2907 http://clucene.svn.sourceforge.net/clucene/?rev=2907&view=rev Author: ustramooner Date: 2008-10-02 14:17:48 +0000 (Thu, 02 Oct 2008) Log Message: ----------- Removed jstreams namespace (and most of the jstreams code, or made them private). Jstreams was the old namespace for Strigi streams. Since we can't have dependencies in the core, using the Strigi namespace is not possible. To use Strigi, use the contribs package. A class will be created soon to easily use Strigi classes. InputStream and Reader are almost exact clones of jstreams (some changes were necessary for Reader use). It will be easy to have a Filter class which passes calls onto Strigi. pass readLine a buffer length BufferedReader is a reader which can be reset. Readers don't have to be resetable. Classes which need reset ability now ask for a BufferedReader Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/analysis/Analyzers.cpp branches/lucene2_3_2/src/core/CLucene/analysis/Analyzers.h branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardAnalyzer.h branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardTokenizer.cpp branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardTokenizer.h branches/lucene2_3_2/src/core/CLucene/document/Field.cpp branches/lucene2_3_2/src/core/CLucene/document/Field.h branches/lucene2_3_2/src/core/CLucene/index/DocumentWriter.cpp branches/lucene2_3_2/src/core/CLucene/queryParser/Lexer.cpp branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.cpp branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParserBase.cpp branches/lucene2_3_2/src/core/CLucene/queryParser/_Lexer.h branches/lucene2_3_2/src/core/CLucene/util/_FastCharStream.h branches/lucene2_3_2/src/core/CLucene.h branches/lucene2_3_2/src/core/CMakeLists.txt branches/lucene2_3_2/src/demo/IndexFiles.cpp Added Paths: ----------- branches/lucene2_3_2/src/core/CLucene/util/CLStreams.h branches/lucene2_3_2/src/core/CLucene/util/_bufferedstream.h branches/lucene2_3_2/src/core/CLucene/util/_streambase.h branches/lucene2_3_2/src/core/CLucene/util/_streambuffer.h Removed Paths: ------------- branches/lucene2_3_2/src/core/CLucene/util/bufferedstream.h branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.cpp branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.h branches/lucene2_3_2/src/core/CLucene/util/inputstreambuffer.h branches/lucene2_3_2/src/core/CLucene/util/jstreamsconfig.h branches/lucene2_3_2/src/core/CLucene/util/streambase.h branches/lucene2_3_2/src/core/CLucene/util/stringreader.h branches/lucene2_3_2/src/core/CLucene/util/subinputstream.h Modified: branches/lucene2_3_2/src/core/CLucene/analysis/Analyzers.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/analysis/Analyzers.cpp 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/analysis/Analyzers.cpp 2008-10-02 14:17:48 UTC (rev 2907) @@ -35,7 +35,7 @@ TCHAR c; offset++; if (bufferIndex >= dataLen) { - dataLen = input->read(ioBuffer, LUCENE_IO_BUFFER_SIZE); + dataLen = input->read(ioBuffer, 1, LUCENE_IO_BUFFER_SIZE ); if (dataLen == -1) dataLen = 0; bufferIndex = 0; @@ -504,7 +504,7 @@ int32_t rd; const TCHAR* buffer=0; while (true) { - rd = input->read(buffer, bufferSize); + rd = input->read(buffer, 1, bufferSize); if (rd == -1) break; token->growBuffer(token->_termTextLen +rd+1); @@ -577,7 +577,7 @@ TCHAR* word = NULL; try { word = _CL_NEWARRAY(TCHAR, LUCENE_DEFAULT_TOKEN_BUFFER_SIZE); - while (reader->readLine(word) > 0) { + while (reader->readLine(word, LUCENE_DEFAULT_TOKEN_BUFFER_SIZE) > 0) { stopTable->insert( STRDUP_TtoT(CL_NS(util)::Misc::wordTrim(word))); } } Modified: branches/lucene2_3_2/src/core/CLucene/analysis/Analyzers.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/analysis/Analyzers.h 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/analysis/Analyzers.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -11,7 +11,7 @@ # pragma once #endif -#include "CLucene/util/Reader.h" +#include "CLucene/util/CLStreams.h" #include "AnalysisHeader.h" #include "CLucene/util/VoidMapSetDefinitions.h" Modified: branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardAnalyzer.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardAnalyzer.h 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardAnalyzer.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -9,7 +9,7 @@ //#include "CLucene/util/VoidMap.h" -CL_CLASS_DEF(util,Reader) +CL_CLASS_DEF(util,BufferedReader) #include "CLucene/analysis/AnalysisHeader.h" #include "CLucene/util/VoidMapSetDefinitions.h" //#include "CLucene/analysis/Analyzers.h" Modified: branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardTokenizer.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardTokenizer.cpp 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardTokenizer.cpp 2008-10-02 14:17:48 UTC (rev 2907) @@ -92,7 +92,7 @@ #define CONTAINS_ANY(sb, ofThese) (_tcscspn(sb.getBuffer(), _T(ofThese)) != static_cast<size_t>(sb.len)) - StandardTokenizer::StandardTokenizer(Reader* reader): + StandardTokenizer::StandardTokenizer(BufferedReader* reader): /* rdPos is zero-based. It starts at -1, and will advance to the first ** position when readChar() is first called. */ rdPos(-1), Modified: branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardTokenizer.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardTokenizer.h 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/analysis/standard/StandardTokenizer.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -11,7 +11,7 @@ #include "../AnalysisHeader.h" //required for Tokenizer #include "StandardTokenizerConstants.h" CL_CLASS_DEF(analysis,Token) -CL_CLASS_DEF(util,Reader) +CL_CLASS_DEF(util,BufferedReader) CL_CLASS_DEF(util,StringBuffer) CL_CLASS_DEF(util,FastCharStream) @@ -53,7 +53,7 @@ CL_NS(util)::FastCharStream* rd; // Constructs a tokenizer for this Reader. - StandardTokenizer(CL_NS(util)::Reader* reader); + StandardTokenizer(CL_NS(util)::BufferedReader* reader); ~StandardTokenizer(); Modified: branches/lucene2_3_2/src/core/CLucene/document/Field.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/document/Field.cpp 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/document/Field.cpp 2008-10-02 14:17:48 UTC (rev 2907) @@ -8,23 +8,12 @@ #include "Field.h" #include "CLucene/util/_StringIntern.h" #include "CLucene/util/StringBuffer.h" -#include "CLucene/util/Reader.h" +#include "CLucene/util/CLStreams.h" +#include "CLucene/analysis/AnalysisHeader.h" CL_NS_USE(util) CL_NS_DEF(document) -/* -struct Field::Internal{ - //const TCHAR* _name; - //TCHAR* _stringValue; - //CL_NS(util)::Reader* _readerValue; - //jstreams::StreamBase<char>* _streamValue; - //void* fieldsData; - - //uint32_t config; - //float_t boost; -};*/ - Field::Field(const TCHAR* Name, Reader* reader, int config): /*_internal(new Internal),*/ lazy(false) { @@ -75,7 +64,7 @@ setConfig(_config); } -Field::Field(const TCHAR* Name, jstreams::StreamBase<char>* Value, int config): +Field::Field(const TCHAR* Name, InputStream* Value, int config): /*_internal(new Internal),*/ lazy(false) { CND_PRECONDITION(Name != NULL, "Name cannot be NULL"); @@ -122,7 +111,7 @@ const TCHAR* Field::name() const { return _name; } ///<returns reference TCHAR* Field::stringValue() const { return (valueType & VALUE_STRING) ? static_cast<TCHAR*>(fieldsData) : NULL; } ///<returns reference Reader* Field::readerValue() const { return (valueType & VALUE_READER) ? static_cast<Reader*>(fieldsData) : NULL; } ///<returns reference -jstreams::StreamBase<char>* Field::streamValue() const { return (valueType & VALUE_STREAM) ? static_cast<jstreams::StreamBase<char>*>(fieldsData) : NULL; } ///<returns reference +InputStream* Field::streamValue() const { return (valueType & VALUE_STREAM) ? static_cast<InputStream*>(fieldsData) : NULL; } ///<returns reference CL_NS(analysis)::TokenStream* Field::tokenStreamValue() const { return (valueType & VALUE_TOKENSTREAM) ? static_cast<CL_NS(analysis)::TokenStream*>(fieldsData) : NULL; } bool Field::isStored() const { return (config & STORE_YES) != 0; } @@ -154,7 +143,7 @@ fieldsData = value; valueType = VALUE_READER; } -void Field::setValue(jstreams::StreamBase<char>* value) { +void Field::setValue(InputStream* value) { _resetValue(); fieldsData = value; valueType = VALUE_STREAM; @@ -311,7 +300,7 @@ Reader* r = static_cast<Reader*>(fieldsData); _CLDELETE(r); } else if (valueType & VALUE_STREAM) { - jstreams::StreamBase<char>* v = static_cast<jstreams::StreamBase<char>*>(fieldsData); + InputStream* v = static_cast<InputStream*>(fieldsData); _CLVDELETE(v); } valueType=VALUE_NONE; Modified: branches/lucene2_3_2/src/core/CLucene/document/Field.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/document/Field.h 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/document/Field.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -18,11 +18,8 @@ */ CL_CLASS_DEF(util,Reader) +CL_CLASS_DEF(util,InputStream) CL_CLASS_DEF(analysis,TokenStream) -namespace jstreams{ - template <class T> - class StreamBase; -} CL_NS_DEF(document) /** @@ -151,7 +148,7 @@ /** * Stream constructor of Field. */ - Field(const TCHAR* name, jstreams::StreamBase<char>* stream, int _config); + Field(const TCHAR* name, CL_NS(util)::InputStream* stream, int _config); Field(const TCHAR* name, int _config); ///<No value, for lazy loading support ~Field(); @@ -173,7 +170,7 @@ /** The value of the field as a String, or null. If null, the String value * or Reader value is used. Exactly one of stringValue(), readerValue() and * streamValue() must be set. */ - jstreams::StreamBase<char>* streamValue() const; + CL_NS(util)::InputStream* streamValue() const; /** The value of the field as a TokesStream, or null. If null, the Reader value, * String value, or binary value is used. Exactly one of stringValue(), @@ -301,7 +298,7 @@ void setValue(CL_NS(util)::Reader* value); /** Expert: change the value of this field. See <a href="#setValue(TCHAR*)">setValue(TCHAR*)</a>. */ - void setValue(jstreams::StreamBase<char>* value) ; + void setValue(CL_NS(util)::InputStream* value) ; /** Expert: change the value of this field. See <a href="#setValue(TCHAR*)">setValue(TCHAR*)</a>. */ void setValue(CL_NS(analysis)::TokenStream* value); Modified: branches/lucene2_3_2/src/core/CLucene/index/DocumentWriter.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/index/DocumentWriter.cpp 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/index/DocumentWriter.cpp 2008-10-02 14:17:48 UTC (rev 2907) @@ -10,7 +10,7 @@ #include "CLucene/store/IndexOutput.h" #include "CLucene/util/Array.h" #include "CLucene/util/Misc.h" -#include "CLucene/util/Reader.h" +#include "CLucene/util/CLStreams.h" #include "CLucene/document/Field.h" #include "CLucene/document/Document.h" #include "_FieldInfos.h" @@ -256,7 +256,7 @@ // this may invalidate the string for the further calls // it may be better to do this via a FilterReader // TODO make a better implementation of this - dataLen = r->read(charBuf, LUCENE_INT32_MAX_SHOULDBE); + dataLen = r->read(charBuf, LUCENE_INT32_MAX_SHOULDBE,0); if (dataLen == -1) dataLen = 0; //todo: would be better to pass the string length, in case Modified: branches/lucene2_3_2/src/core/CLucene/queryParser/Lexer.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/queryParser/Lexer.cpp 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/queryParser/Lexer.cpp 2008-10-02 14:17:48 UTC (rev 2907) @@ -10,7 +10,7 @@ #include "QueryToken.h" #include "_Lexer.h" -#include "CLucene/util/Reader.h" +#include "CLucene/util/CLStreams.h" #include "CLucene/util/StringBuffer.h" #include "CLucene/util/_FastCharStream.h" @@ -46,7 +46,7 @@ } -Lexer::Lexer(QueryParserBase* queryparser, Reader* source) { +Lexer::Lexer(QueryParserBase* queryparser, BufferedReader* source) { //Func - Constructor // Initializes a new instance of the Lexer class with the specified // TextReader to lex. Modified: branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.cpp 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.cpp 2008-10-02 14:17:48 UTC (rev 2907) @@ -8,7 +8,7 @@ #include "QueryParser.h" #include "CLucene/analysis/AnalysisHeader.h" -#include "CLucene/util/Reader.h" +#include "CLucene/util/CLStreams.h" #include "CLucene/search/SearchHeader.h" #include "CLucene/search/BooleanClause.h" #include "CLucene/search/Query.h" @@ -72,7 +72,7 @@ CND_PRECONDITION(query != NULL, "query is NULL"); //Instantie a Stringer that can read the query string - Reader* r = _CLNEW StringReader(query); + BufferedReader* r = _CLNEW StringReader(query); //Check to see if r has been created properly CND_CONDITION(r != NULL, "Could not allocate memory for StringReader r"); @@ -91,7 +91,7 @@ return ret; } - Query* QueryParser::parse(Reader* reader){ + Query* QueryParser::parse(BufferedReader* reader){ //Func - Returns a parsed Query instance //Pre - reader contains a valid reference to a Reader and manages the query string //Post - A parsed Query instance has been returned or Modified: branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -9,7 +9,7 @@ //#include "CLucene/analysis/AnalysisHeader.h" -CL_CLASS_DEF(util,Reader) +CL_CLASS_DEF(util,BufferedReader) //#include "CLucene/search/SearchHeader.h" CL_CLASS_DEF(index,Term) CL_CLASS_DEF(analysis,Analyzer) @@ -267,7 +267,7 @@ * <param name="reader">The TextReader value to be parsed.</param> * <returns>A parsed Query instance.</returns> */ - virtual CL_NS(search)::Query* parse(CL_NS(util)::Reader* reader); + virtual CL_NS(search)::Query* parse(CL_NS(util)::BufferedReader* reader); /** * Returns a new instance of the Query class with a specified query, field and Modified: branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParserBase.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParserBase.cpp 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParserBase.cpp 2008-10-02 14:17:48 UTC (rev 2907) @@ -16,7 +16,7 @@ #include "CLucene/search/BooleanQuery.h" #include "CLucene/analysis/AnalysisHeader.h" -#include "CLucene/util/Reader.h" +#include "CLucene/util/CLStreams.h" #include "CLucene/search/SearchHeader.h" #include "CLucene/search/BooleanClause.h" #include "CLucene/search/Query.h" Modified: branches/lucene2_3_2/src/core/CLucene/queryParser/_Lexer.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/queryParser/_Lexer.h 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/queryParser/_Lexer.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -9,7 +9,7 @@ CL_CLASS_DEF(util,FastCharStream) -CL_CLASS_DEF(util,Reader) +CL_CLASS_DEF(util,BufferedReader) CL_CLASS_DEF(util,StringBuffer) //#include "TokenList.h" @@ -34,7 +34,7 @@ // Initializes a new instance of the Lexer class with the specified // TextReader to lex. - Lexer(QueryParserBase* queryparser, CL_NS(util)::Reader* source); + Lexer(QueryParserBase* queryparser, CL_NS(util)::BufferedReader* source); //Breaks the input stream onto the tokens list tokens void Lex(TokenList *tokens); Added: branches/lucene2_3_2/src/core/CLucene/util/CLStreams.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/CLStreams.h (rev 0) +++ branches/lucene2_3_2/src/core/CLucene/util/CLStreams.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -0,0 +1,233 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#ifndef _lucene_util_CLStreams_ +#define _lucene_util_CLStreams_ + +CL_NS_DEF(util) + +template <typename T> +class CLUCENE_EXPORT CLStream{ +public: + virtual ~CLStream(){} + + inline int read(){ + const T* buffer; + const int32_t nread = read(buffer,1, 1); + if ( nread < 0 ) + return -1; + else + return buffer[0]; + } + + /** Read one line, return the length of the line read. + * If the string is longer than len, only len of that line will be copied + */ + inline int32_t readLine(T* buffer, size_t len){ + size_t i = 0; + while (true && i<len-1) { + int32_t b = read(); + if (b < 1) + break; + if (b == '\n' || b == '\r') { + if (i > 0) + break; + else + continue; + } + buffer[i++] = b; + } + buffer[i] = 0; + return i; + } + + /** + * @brief Reads items from the stream and sets @p start to point to + * the first item that was read. + * + * Note: unless stated otherwise in the documentation for that method, + * this pointer will no longer be valid after calling another method of + * this class. The pointer will also no longer be valid after the class + * is destroyed. + * + * At least @p min items will be read from the stream, unless an error occurs + * or the end of the stream is reached. Under no circumstances will more than + * @p max items be read. + * + * If the end of the stream is reached before @p min items are read, the + * read is still considered successful and the number of items read will + * be returned. + * + * @param start pointer passed by reference that will be set to point to + * the retrieved array of items. If the end of the stream + * is encountered or an error occurs, the value of @p start + * is undefined + * @param min the minimal number of items to read from the stream. This + * value should be larger than 0. If it is 0 or smaller, the + * result is undefined + * @param max the maximal number of items to read from the stream. + * If this value is smaller than @p min, there is no limit on + * the number of items that can be read + * @return the number of items that were read. @c -1 is returned if + * end of the stream has already been reached. An error is thrown + * if an error occurs. + **/ + virtual int32_t read(const T*& start, int32_t min, int32_t max) = 0; + /** + * @brief Skip @p ntoskip items. + * + * If an error occurs, or the end of the stream is encountered, fewer + * than @p ntoskip items may be skipped. This can be checked by comparing + * the return value to @p ntoskip. + * + * Calling this function invalidates the data pointer that was obtained from + * StreamBase::read. + * + * @param ntoskip the number of items that should be skipped + * @return the number of items skipped + **/ + virtual int64_t skip(int64_t ntoskip) = 0; + /** + * @brief Get the current position in the stream. + * The value obtained from this function can be used to reset the stream. + **/ + virtual int64_t position() = 0; + + virtual size_t size() = 0; +}; + +template <class T> +class CLUCENE_EXPORT BufferedStream{ +public: + /** + * @brief Repositions this stream to a given position. + * + * A call to reset is only guaranteed to be successful when + * the requested position lies within the segment of a stream + * corresponding to a valid pointer obtained from read. + * In this case, the pointer will not be invalidated. + * + * Calling this function invalidates the data pointer that was obtained from + * StreamBase::read unless the conditions outlined above apply. + * + * To read n items, leaving the stream at the same position as before, you + * can do the following: + * @code + * int64_t start = stream.position(); + * if ( stream.read(data, min, max) > 0 ) { + * stream.reset(start); + * // The data pointer is still valid here + * } + * @endcode + * + * @param pos the position in the stream you want to go to, relative to + * the start of the stream + * @return the new position in the stream + **/ + virtual int64_t reset(int64_t) = 0; + /** + * @brief Sets the minimum size of the buffer + */ + virtual void setMinBufSize(int32_t s) = 0; +}; + +class BufferedReader; +class CLUCENE_EXPORT Reader: public CLStream<TCHAR>{ +public: + virtual BufferedReader* __asBufferedReader(){ return NULL; } +}; +class CLUCENE_EXPORT BufferedReader: public Reader, public BufferedStream<TCHAR>{ +public: + _CL_DEPRECATED( setMinBufSize ) int64_t mark(int32_t readAheadlimit){ + this->setMinBufSize(readAheadlimit); + return this->position(); + } + BufferedReader* __asBufferedReader(){ return this; } +}; +class CLUCENE_EXPORT InputStream: public CLStream<signed char>{}; +class CLUCENE_EXPORT BufferedInputStream: public InputStream, public BufferedStream<signed char>{}; + + +class CLUCENE_EXPORT StringReader: public BufferedReader{ + TCHAR* value; + bool ownValue; + int64_t pos; +protected: + size_t m_size; +public: + StringReader ( const TCHAR* value, const int32_t length = -1 ); + StringReader ( TCHAR* value, const int32_t length, bool copyData = true ); + virtual ~StringReader(); + + int32_t read(const TCHAR*& start, int32_t min, int32_t max); + int64_t position(); + int64_t reset(int64_t); + int64_t skip(int64_t ntoskip); + void setMinBufSize(int32_t s); + size_t size(); +}; + +/** +* A helper class which constructs a FileReader with a specified +* simple encodings, or a given inputstreamreader +*/ +class CLUCENE_EXPORT FileInputStream: public BufferedInputStream { + class Internal; + Internal* internal; +protected: + void init(InputStream *i, int encoding); +public: + LUCENE_STATIC_CONSTANT(int32_t, DEFAULT_BUFFER_SIZE=4096); + FileInputStream ( const char* path, int32_t buflen = -1 ); + virtual ~FileInputStream (); + + int32_t read(const signed char*& start, int32_t min, int32_t max); + int64_t position(); + int64_t reset(int64_t); + int64_t skip(int64_t ntoskip); + size_t size(); + void setMinBufSize(int32_t minbufsize); +}; + +class CLUCENE_EXPORT SimpleInputStreamReader: public BufferedReader{ + class Internal; + Internal* internal; +protected: + void init(InputStream *i, int encoding); +public: + enum{ + ASCII=1, + UTF8=2, + UCS2_LE=3 + }; + + SimpleInputStreamReader(); + virtual ~SimpleInputStreamReader(); + + int32_t read(const TCHAR*& start, int32_t min, int32_t max); + int64_t position(); + int64_t reset(int64_t); + int64_t skip(int64_t ntoskip); + void setMinBufSize(int32_t s); + size_t size(); +}; + +/** +* A helper class which constructs a FileReader with a specified +* simple encodings, or a given inputstreamreader. +* It is recommended that you use the contribs package for proper +* decoding using iconv. This class is provided only as a dependency-less +* replacement. +*/ +class CLUCENE_EXPORT FileReader: public SimpleInputStreamReader{ +public: + FileReader(const char* path, int encoding, int32_t buflen = -1); + FileReader(const char* path, const char* encoding, int32_t buflen = -1); + virtual ~FileReader(); +}; + +CL_NS_END +#endif Modified: branches/lucene2_3_2/src/core/CLucene/util/_FastCharStream.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/_FastCharStream.h 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/util/_FastCharStream.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -8,7 +8,7 @@ #define _lucene_util_FastCharStream_ -CL_CLASS_DEF(util,Reader) +CL_CLASS_DEF(util,BufferedReader) CL_NS_DEF(util) @@ -24,10 +24,10 @@ // read character from stream return false on error void readChar(TCHAR &); public: - Reader* input; + BufferedReader* input; /// Initializes a new instance of the FastCharStream class LUCENE_EXPORT. - FastCharStream(Reader* reader); + FastCharStream(BufferedReader* reader); ~FastCharStream(); /// Returns the next TCHAR from the stream. Copied: branches/lucene2_3_2/src/core/CLucene/util/_bufferedstream.h (from rev 2905, branches/lucene2_3_2/src/core/CLucene/util/bufferedstream.h) =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/_bufferedstream.h (rev 0) +++ branches/lucene2_3_2/src/core/CLucene/util/_bufferedstream.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -0,0 +1,179 @@ +#ifndef STRIGI_BUFFEREDSTREAM_H +#define STRIGI_BUFFEREDSTREAM_H + +#include "_streambase.h" +#include "_streambuffer.h" +#include <cassert> + +namespace jstreams { + +/** + * @brief Abstract class providing a buffered input stream. + * + * You can inherit this class to provide buffered access to a + * resource. You just need to implement fillBuffer, and + * BufferedStream will do the rest. + */ +template <class T> +class BufferedStream : public StreamBase<T> { +private: + StreamBuffer<T> buffer; + bool finishedWritingToBuffer; + + void writeToBuffer(int32_t minsize, int32_t maxsize); +protected: + /** + * @brief Fill the buffer with the provided data + * + * This function should be implemented by subclasses. + * It should write up to @p space characters from the + * stream to the buffer position pointed to by @p start. + * + * If the end of the stream is encountered, -1 should be + * returned. + * + * If an error occurs, the status should be set to Error, + * an error message should be set and -1 should be returned. + * + * You should @em not call this function yourself. + * + * @param start where the data should be written to + * @param space the maximum amount of data to write + * @return Number of characters written, or -1 on error + **/ + virtual int32_t fillBuffer(T* start, int32_t space) = 0; + /** + * @brief Resets the buffer, allowing it to be used again + * + * This function resets the buffer, allowing it to be re-used. + */ + void resetBuffer() { + StreamBase<T>::m_size = -1; + StreamBase<T>::m_position = 0; + StreamBase<T>::m_error.assign(""); + StreamBase<T>::m_status = Ok; + buffer.readPos = buffer.start; + buffer.avail = 0; + finishedWritingToBuffer = false; + } + /** + * @brief Sets the minimum size of the buffer + */ + void setMinBufSize(int32_t s) { + buffer.makeSpace(s); + } + BufferedStream<T>(); +public: + int32_t read(const T*& start, int32_t min, int32_t max); + int64_t reset(int64_t pos); + virtual int64_t skip(int64_t ntoskip); +}; + + +/** Abstract class for a buffered stream of bytes */ +typedef BufferedStream<signed char> BufferedInputStream; + +/** Abstract class for a buffered stream of Unicode characters */ +typedef BufferedStream<wchar_t> BufferedReader; + + +template <class T> +BufferedStream<T>::BufferedStream() { + finishedWritingToBuffer = false; +} + +template <class T> +void +BufferedStream<T>::writeToBuffer(int32_t ntoread, int32_t maxread) { + int32_t missing = ntoread - buffer.avail; + int32_t nwritten = 0; + while (missing > 0 && nwritten >= 0) { + int32_t space; + space = buffer.makeSpace(missing); + if (maxread >= ntoread && space > maxread) { + space = maxread; + } + T* start = buffer.readPos + buffer.avail; + nwritten = fillBuffer(start, space); + assert(StreamBase<T>::m_status != Eof); + if (nwritten > 0) { + buffer.avail += nwritten; + missing = ntoread - buffer.avail; + } + } + if (nwritten < 0) { + finishedWritingToBuffer = true; + } +} +template <class T> +int32_t +BufferedStream<T>::read(const T*& start, int32_t min, int32_t max) { + if (StreamBase<T>::m_status == Error) return -2; + if (StreamBase<T>::m_status == Eof) return -1; + + // do we need to read data into the buffer? + if (min > max) max = 0; + if (!finishedWritingToBuffer && min > buffer.avail) { + // do we have enough space in the buffer? + writeToBuffer(min, max); + if (StreamBase<T>::m_status == Error) return -2; + } + + int32_t nread = buffer.read(start, max); + + StreamBase<T>::m_position += nread; + if (StreamBase<T>::m_position > StreamBase<T>::m_size + && StreamBase<T>::m_size > 0) { + // error: we read more than was specified in size + // this is an error because all dependent code might have been labouring + // under a misapprehension + StreamBase<T>::m_status = Error; + StreamBase<T>::m_error = "Stream is longer than specified."; + nread = -2; + } else if (StreamBase<T>::m_status == Ok && buffer.avail == 0 + && finishedWritingToBuffer) { + StreamBase<T>::m_status = Eof; + if (StreamBase<T>::m_size == -1) { + StreamBase<T>::m_size = StreamBase<T>::m_position; + } + // save one call to read() by already returning -1 if no data is there + if (nread == 0) nread = -1; + } + return nread; +} +template <class T> +int64_t +BufferedStream<T>::reset(int64_t newpos) { + assert(newpos >= 0); + if (StreamBase<T>::m_status == Error) return -2; + // check to see if we have this position + int64_t d = StreamBase<T>::m_position - newpos; + if (buffer.readPos - d >= buffer.start && -d < buffer.avail) { + StreamBase<T>::m_position -= d; + buffer.avail += (int32_t)d; + buffer.readPos -= d; + StreamBase<T>::m_status = Ok; + } + return StreamBase<T>::m_position; +} +template <class T> +int64_t +BufferedStream<T>::skip(int64_t ntoskip) { + const T *begin; + int32_t nread; + int64_t skipped = 0; + while (ntoskip) { + int32_t step = (int32_t)((ntoskip > buffer.size) ?buffer.size :ntoskip); + nread = read(begin, 1, step); + if (nread <= 0) { + return skipped; + } + ntoskip -= nread; + skipped += nread; + } + return skipped; +} + +} // end namespace Strigi + +#endif Property changes on: branches/lucene2_3_2/src/core/CLucene/util/_bufferedstream.h ___________________________________________________________________ Added: svn:mergeinfo + Copied: branches/lucene2_3_2/src/core/CLucene/util/_streambase.h (from rev 2905, branches/lucene2_3_2/src/core/CLucene/util/streambase.h) =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/_streambase.h (rev 0) +++ branches/lucene2_3_2/src/core/CLucene/util/_streambase.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -0,0 +1,212 @@ + +#ifndef STRIGI_STREAMBASE_H +#define STRIGI_STREAMBASE_H + +#include <stdio.h> +#include <string> + +#define INT32MAX 0x7FFFFFFFL + +namespace jstreams { + +/** Used to indicate the current status of a Stream */ +enum StreamStatus { + Ok /**< Stream is capable of being read from */, + Eof /**< The end of the Stream has been reached */, + Error /**< An error occurred. Use error() to find out more information */ +}; + +// java mapping: long=int64, int=int32, byte=uint8_t +/** + * The base of all Streams. Do not inherit directly from this class, + * but from (an instance of) StreamBase + * + * This class contains all the non-virtual StreamBase methods + * that don't depend on a specific Stream type + * + * Developer comment: This is needed because of win32 compilation. + * When we want to access a function outside a lib, we have to export them, + * but we can't export the template class because this would be somewhat + * stupid / does not work by design :) + * Because of this I've introduced this StreamBaseBase class + */ +class StreamBaseBase { //krazy:exclude=dpointer +protected: + /** The size of the stream (-1 if unknown) */ + int64_t m_size; + /** The position of the stream */ + int64_t m_position; + /** + * @brief String representation of the last error, or + * an empty string otherwise + */ + std::string m_error; + /** The status of the stream - see StreamStatus */ + StreamStatus m_status; +public: + /** + * @brief Constructor: initialises everything to sane defaults + **/ + StreamBaseBase() :m_size(-1), m_position(0), m_status(Ok) {} + /** + * @brief Destructor + **/ + virtual ~StreamBaseBase() {} + /** + * @brief Return a string representation of the last error. + * If no error has occurred, an empty string is returned. + **/ + const char* error() const { return m_error.c_str(); } + /** + * @brief Return the status of the stream. + **/ + StreamStatus status() const { return m_status; } + /** + * @brief Get the current position in the stream. + * The value obtained from this function can be used to reset the stream. + **/ + int64_t position() const { return m_position; } + /** + * @brief Return the size of the stream. + * + * The size of the stream is always known if the end of the stream + * has been reached. In all other cases, this may return -1 to + * indicate the size of the stream is unknown. + * + * @return the size of the stream, if it is known, or -1 if the size + * of the stream is unknown + **/ + int64_t size() const { return m_size; } +}; + +/** + * @brief Base class for stream read access to a data source. + * + * This class is based on the interface java.io.InputStream. It provides + * a uniform interface for accessing streamed resources. + * + * The main difference with the Java equivalent is a performance improvement. + * When reading data, data is not copied into a buffer provided by the caller, + * but a pointer to the read data is provided. This makes this interface + * especially useful for deriving from it and implementing filters or + * transformers. + */ +template <class T> +class StreamBase : public StreamBaseBase { +public: + StreamBase() { } + virtual ~StreamBase(){} + /** + * @brief Reads items from the stream and sets @p start to point to + * the first item that was read. + * + * Note: unless stated otherwise in the documentation for that method, + * this pointer will no longer be valid after calling another method of + * this class. The pointer will also no longer be valid after the class + * is destroyed. + * + * The functions inherited from StreamBaseBase do not invalidate the pointer. + * + * At least @p min items will be read from the stream, unless an error occurs + * or the end of the stream is reached. Under no circumstances will more than + * @p max items be read. + * + * If the end of the stream is reached before @p min items are read, the + * read is still considered successful and the number of items read will + * be returned. + * + * @param start pointer passed by reference that will be set to point to + * the retrieved array of items. If the end of the stream + * is encountered or an error occurs, the value of @p start + * is undefined + * @param min the minimal number of items to read from the stream. This + * value should be larger than 0. If it is 0 or smaller, the + * result is undefined + * @param max the maximal number of items to read from the stream. + * If this value is smaller than @p min, there is no limit on + * the number of items that can be read + * @return the number of items that were read. @c -1 is returned if + * end of the stream has already been reached. @c -2 is returned + * if an error has occurred + **/ + virtual int32_t read(const T*& start, int32_t min, int32_t max) = 0; + /** + * @brief Skip @p ntoskip items. + * + * If an error occurs, or the end of the stream is encountered, fewer + * than @p ntoskip items may be skipped. This can be checked by comparing + * the return value to @p ntoskip. + * + * Calling this function invalidates the data pointer that was obtained from + * StreamBase::read. + * + * @param ntoskip the number of items that should be skipped + * @return the number of items skipped + **/ + virtual int64_t skip(int64_t ntoskip); + /** + * @brief Repositions this stream to a given position. + * + * A call to StreamBase::reset is only guaranteed to be successful when + * the requested position lies within the segment of a stream + * corresponding to a valid pointer obtained from StreamBase::read. + * In this case, the pointer will not be invalidated. + * + * Calling this function invalidates the data pointer that was obtained from + * StreamBase::read unless the conditions outlined above apply. + * + * To read n items, leaving the stream at the same position as before, you + * can do the following: + * @code + * int64_t start = stream.position(); + * if ( stream.read(data, min, max) > 0 ) { + * stream.reset(start); + * // The data pointer is still valid here + * } + * @endcode + * + * @param pos the position in the stream you want to go to, relative to + * the start of the stream + * @return the new position in the stream + **/ + virtual int64_t reset(int64_t pos) = 0; +}; + + +/** Abstract class for a stream of bytes */ +typedef StreamBase<char> InputStream; + +/** Abstract class for a stream of Unicode characters */ +typedef StreamBase<wchar_t> Reader; + + +template <class T> +int64_t +StreamBase<T>::skip(int64_t ntoskip) { + const T* begin; + int32_t nread; + int64_t skipped = 0; + while (ntoskip > 0) { + // make sure we do not overflow uint32_t + int32_t maxstep = (int32_t)((ntoskip > 10000000) + ?10000000 :ntoskip); + // the default implementation is to simply read the data that we want + // to skip + nread = read(begin, 1, maxstep); + if (nread < -1 ) { + // an error occurred + return nread; + } else if (nread < 1) { + // the end of the stream was encountered + ntoskip = 0; + } else { + skipped += nread; + ntoskip -= nread; + } + } + return skipped; +} + +} // end namespace Strigi + +#endif Property changes on: branches/lucene2_3_2/src/core/CLucene/util/_streambase.h ___________________________________________________________________ Added: svn:mergeinfo + Added: branches/lucene2_3_2/src/core/CLucene/util/_streambuffer.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/_streambuffer.h (rev 0) +++ branches/lucene2_3_2/src/core/CLucene/util/_streambuffer.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -0,0 +1,162 @@ + +#ifndef STRIGI_STREAMBUFFER_H +#define STRIGI_STREAMBUFFER_H + +#include <cstdlib> +#include <cstring> + +namespace jstreams { + +/** + * @internal + * @brief Provides a buffer for the use of BufferedStream + */ +template <class T> +class StreamBuffer { +private: +public: + /** + * @internal + * @brief Pointer to the start of the buffer. + */ + T* start; + /** + * @internal + * @brief Size of the buffer. + * + * Size of the memory pointed to by @p start, + * in multiples of sizeof(T) + */ + int32_t size; + /** + * @internal + * @brief Pointer to the current position the buffer. + */ + T* readPos; + /** + * @internal + * @brief The amount of data available in the buffer. + * + * The size of the used memory in the buffer, starting + * from @p readPos. @p readPos + @p avail must be + * greater than @p start + @p size. + */ + int32_t avail; + + /** + * @internal + * @brief Constructor: initialises members to sane defaults. + */ + StreamBuffer(); + /** + * @internal + * @brief Destructor: frees the memory used by the buffer. + */ + ~StreamBuffer(); + /** + * @internal + * @brief Sets the size of the buffer, allocating the necessary memory + * + * @param size the size that the buffer should be, in multiples + * of sizeof(T) + */ + void setSize(int32_t size); + /** + * @internal + * @brief Read data from the buffer + * + * Sets @p start to point to the data, starting + * at the item of data following the last item + * of data read. + * + * @param start pointer passed by reference. It will + * be set to point to the data read from the buffer + * @param max the maximum amount of data to read from + * the buffer + * @return the size of the data pointed to by @p start + * (always less than or equal to @p max) + */ + int32_t read(const T*& start, int32_t max=0); + + /** + * @internal + * @brief Prepares the buffer for a new write. + * + * This function invalidates any pointers + * previously obtained from read. + * + * @return the number of available places + **/ + int32_t makeSpace(int32_t needed); +}; + +template <class T> +StreamBuffer<T>::StreamBuffer() { + readPos = start = 0; + size = avail = 0; +} +template <class T> +StreamBuffer<T>::~StreamBuffer() { + std::free(start); +} +template <class T> +void +StreamBuffer<T>::setSize(int32_t size) { + // store pointer information + int32_t offset = readPos - start; + + // allocate memory in the buffer + start = (T*)std::realloc(start, size*sizeof(T)); + this->size = size; + + // restore pointer information + readPos = start + offset; +} +template <class T> +int32_t +StreamBuffer<T>::makeSpace(int32_t needed) { + // determine how much space is available for writing + int32_t space = size - (readPos - start) - avail; + if (space >= needed) { + // there's enough space + return space; + } + + if (avail) { + if (readPos != start) { +// printf("moving\n"); + // move data to the start of the buffer + std::memmove(start, readPos, avail*sizeof(T)); + space += readPos - start; + readPos = start; + } + } else { + // we may start writing at the start of the buffer + readPos = start; + space = size; + } + if (space >= needed) { + // there's enough space now + return space; + } + + // still not enough space, we have to allocate more +// printf("resize %i %i %i %i %i\n", avail, needed, space, size + needed - space, size); + setSize(size + needed - space); + return needed; +} +template <class T> +int32_t +StreamBuffer<T>::read(const T*& start, int32_t max) { + start = readPos; + if (max <= 0 || max > avail) { + max = avail; + } + readPos += max; + avail -= max; + return max; +} + +} // end namespace Strigi + +#endif Deleted: branches/lucene2_3_2/src/core/CLucene/util/bufferedstream.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/bufferedstream.h 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/util/bufferedstream.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -1,140 +0,0 @@ -/*------------------------------------------------------------------------------ -* Copyright (C) 2003-2006 Jos van den Oever -* -* Distributable under the terms of either the Apache License (Version 2.0) or -* the GNU Lesser General Public License, as specified in the COPYING file. -------------------------------------------------------------------------------*/ -/* This file is part of Strigi Desktop Search - * - * Copyright (C) 2006 Jos van den Oever <jo...@va...> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public License - * along with this library; see the file COPYING.LIB. If not, write to - * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ -#ifndef BUFFEREDSTREAM_H -#define BUFFEREDSTREAM_H - -#include "streambase.h" -#include "inputstreambuffer.h" -#include <cassert> - -namespace jstreams { - -template <class T> -class CLUCENE_EXPORT BufferedInputStream : public StreamBase<T> { -private: - bool finishedWritingToBuffer; - InputStreamBuffer<T> buffer; - - void writeToBuffer(int32_t ntoread){ - int32_t missing = ntoread - buffer.avail; - int32_t nwritten = 0; - while (missing > 0 && nwritten >= 0) { - int32_t space; - space = buffer.makeSpace(missing); - T* start = buffer.readPos + buffer.avail; - nwritten = fillBuffer(start, space); - assert(StreamBase<T>::status != Eof); - if (nwritten > 0) { - buffer.avail += nwritten; - missing = ntoread - buffer.avail; - } - } - if (nwritten < 0) { - finishedWritingToBuffer = true; - } - } -protected: - /** - * This function must be implemented by the subclasses. - * It should write a maximum of @p space characters at the buffer - * position pointed to by @p start. If no more data is avaiable due to - * end of file, -1 should be returned. If an error occurs, the status - * should be set to Error, an error message should be set and the function - * must return -1. - **/ - virtual int32_t fillBuffer(T* start, int32_t space) = 0; - // this function might be useful if you want to reuse a bufferedstream - void resetBuffer() {printf("implement 'resetBuffer'\n");} - BufferedInputStream<T>(){ - finishedWritingToBuffer = false; - } -public: - int32_t read(const T*& start, int32_t min, int32_t max){ - if (StreamBase<T>::status == Error) return -2; - if (StreamBase<T>::status == Eof) return -1; - - // do we need to read data into the buffer? - if (!finishedWritingToBuffer && min > buffer.avail) { - // do we have enough space in the buffer? - writeToBuffer(min); - if (StreamBase<T>::status == Error) return -2; - } - - int32_t nread = buffer.read(start, max); - - BufferedInputStream<T>::position += nread; - if (BufferedInputStream<T>::position > BufferedInputStream<T>::size - && BufferedInputStream<T>::size > 0) { - // error: we read more than was specified in size - // this is an error because all dependent code might have been labouring - // under a misapprehension - BufferedInputStream<T>::status = Error; - BufferedInputStream<T>::error = "Stream is longer than specified."; - nread = -2; - } else if (BufferedInputStream<T>::status == Ok && buffer.avail == 0 - && finishedWritingToBuffer) { - BufferedInputStream<T>::status = Eof; - if (BufferedInputStream<T>::size == -1) { - BufferedInputStream<T>::size = BufferedInputStream<T>::position; - } - // save one call to read() by already returning -1 if no data is there - if (nread == 0) nread = -1; - } - return nread; - } - int64_t reset(int64_t newpos){ - if (StreamBase<T>::status == Error) return -2; - // check to see if we have this position - int64_t d = BufferedInputStream<T>::position - newpos; - if (buffer.readPos - d >= buffer.start && -d < buffer.avail) { - BufferedInputStream<T>::position -= d; - buffer.avail += (int32_t)d; - buffer.readPos -= d; - StreamBase<T>::status = Ok; - } - return StreamBase<T>::position; - } - virtual int64_t skip(int64_t ntoskip){ - const T *begin; - int32_t nread; - int64_t skipped = 0; - while (ntoskip) { - int32_t step = (int32_t)((ntoskip > buffer.size) ?buffer.size :ntoskip); - nread = read(begin, 1, step); - if (nread <= 0) { - return skipped; - } - ntoskip -= nread; - skipped += nread; - } - return skipped; - } -}; - - -} - -#endif Deleted: branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.cpp 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.cpp 2008-10-02 14:17:48 UTC (rev 2907) @@ -1,102 +0,0 @@ -/*------------------------------------------------------------------------------ -* Copyright (C) 2003-2006 Jos van den Oever -* -* Distributable under the terms of either the Apache License (Version 2.0) or -* the GNU Lesser General Public License, as specified in the COPYING file. -------------------------------------------------------------------------------*/ -/* This file is part of Strigi Desktop Search - * - * Copyright (C) 2006 Jos van den Oever <jo...@va...> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public License - * along with this library; see the file COPYING.LIB. If not, write to - * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ -#include "jstreamsconfig.h" -#include "fileinputstream.h" -#include <cerrno> -#include <cstring> - -#ifdef _CL_HAVE_STRING_H - #include <string.h> -#endif - -namespace jstreams { - -FileInputStream::FileInputStream(const char *filepath, int32_t buffersize) { - // try to open the file for reading - file = fopen(filepath, "rb"); - this->filepath = filepath; - if (file == 0) { - // handle error - error = "Could not read file '"; - error += filepath; - error += "': "; - error += strerror(errno); - status = Error; - return; - } - // determine file size. if the stream is not seekable, the size will be -1 - fseek(file, 0, SEEK_END); - size = ftell(file); - fseek(file, 0, SEEK_SET); - - // if the file has size 0, make sure that it's really empty - // this is useful for filesystems like /proc that report files as size 0 - // for files that do contain content - if (size == 0) { - char dummy[1]; - size_t n = fread(dummy, 1, 1, file); - if (n == 1) { - size = -1; - fseek(file, 0, SEEK_SET); - } else { - fclose(file); - file = 0; - return; - } - } - - // allocate memory in the buffer - int32_t bufsize = (size <= buffersize) ?size+1 :buffersize; - mark(bufsize); -} -FileInputStream::~FileInputStream() { - if (file) { - if (fclose(file)) { - // handle error - error = "Could not close file '" + filepath + "'."; - } - } -} -int32_t -FileInputStream::fillBuffer(char* start, int32_t space) { - if (file == 0) return -1; - // read into the buffer - int32_t nwritten = fread(start, 1, space, file); - // check the file stream status - if (ferror(file)) { - error = "Could not read from file '" + filepath + "'."; - fclose(file); - file = 0; - status = Error; - return -1; - } - if (feof(file)) { - fclose(file); - file = 0; - } - return nwritten; -} -} Deleted: branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.h 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/util/fileinputstream.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -1,49 +0,0 @@ -/*------------------------------------------------------------------------------ -* Copyright (C) 2003-2006 Jos van den Oever -* -* Distributable under the terms of either the Apache License (Version 2.0) or -* the GNU Lesser General Public License, as specified in the COPYING file. -------------------------------------------------------------------------------*/ -/* This file is part of Strigi Desktop Search - * - * Copyright (C) 2006 Jos van den Oever <jo...@va...> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public License - * along with this library; see the file COPYING.LIB. If not, write to - * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ -#ifndef FILEINPUTSTREAM_H -#define FILEINPUTSTREAM_H - -#include <stdio.h> -#include "bufferedstream.h" - -namespace jstreams { - -class CLUCENE_EXPORT FileInputStream : public BufferedInputStream<char> { -private: - FILE *file; - std::string filepath; - -public: - LUCENE_STATIC_CONSTANT(int32_t, defaultBufferSize=1048576); - FileInputStream(const char *filepath, int32_t buffersize=defaultBufferSize); - ~FileInputStream(); - int32_t fillBuffer(char* start, int32_t space); -}; - -} // end namespace jstreams - -#endif - Deleted: branches/lucene2_3_2/src/core/CLucene/util/inputstreambuffer.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/inputstreambuffer.h 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/util/inputstreambuffer.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -1,113 +0,0 @@ -/*------------------------------------------------------------------------------ -* Copyright (C) 2003-2006 Jos van den Oever -* -* Distributable under the terms of either the Apache License (Version 2.0) or -* the GNU Lesser General Public License, as specified in the COPYING file. -------------------------------------------------------------------------------*/ -/* This file is part of Strigi Desktop Search - * - * Copyright (C) 2006 Jos van den Oever <jo...@va...> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public License - * along with this library; see the file COPYING.LIB. If not, write to - * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ -#ifndef INPUTSTREAMBUFFER_H -#define INPUTSTREAMBUFFER_H - -#include <stdlib.h> - -namespace jstreams { - -template <class T> -class CLUCENE_EXPORT InputStreamBuffer { -private: -public: - T* start; - int32_t size; - T* readPos; - int32_t avail; - - InputStreamBuffer(){ - readPos = start = 0; - size = avail = 0; - } - ~InputStreamBuffer(){ - free(start); - } - void setSize(int32_t size){ - // store pointer information - int32_t offset = (int32_t)(readPos - start); - - // allocate memory in the buffer - if ( start == 0 ) - start = (T*)malloc(size*sizeof(T)); - else - start = (T*)realloc(start, size*sizeof(T)); - this->size = size; - - // restore pointer information - readPos = start + offset; - } - int32_t read(const T*& start, int32_t max=0){ - start = readPos; - if (max <= 0 || max > avail) { - max = avail; - } - readPos += max; - avail -= max; - return max; - } - - /** - * This function prepares the buffer for a new write. - * returns the number of available places. - **/ - int32_t makeSpace(int32_t needed){ - // determine how much space is available for writing - int32_t space = size - ((int32_t)(readPos - start)) - avail; - if (space >= needed) { - // there's enough space - return space; - } - - if (avail) { - if (readPos != start) { - // printf("moving\n"); - // move data to the start of the buffer - memmove(start, readPos, avail*sizeof(T)); - space += (int32_t)(readPos - start); - readPos = start; - } - } else { - // we may start writing at the start of the buffer - readPos = start; - space = size; - } - if (space >= needed) { - // there's enough space now - return space; - } - - // still not enough space, we have to allocate more - // printf("resize %i %i %i %i %i\n", avail, needed, space, size + needed - space, size); - setSize(size + needed - space); - return needed; - } -}; - - -} // end namespace jstreams - -#endif Deleted: branches/lucene2_3_2/src/core/CLucene/util/jstreamsconfig.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/util/jstreamsconfig.h 2008-09-30 14:54:57 UTC (rev 2906) +++ branches/lucene2_3_2/src/core/CLucene/util/jstreamsconfig.h 2008-10-02 14:17:48 UTC (rev 2907) @@ -1,9 +0,0 @@ -/*------------------------------------------------------------------------------ -* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team -* -* Distributable under the terms of either the Apache License (Version 2.0) or -* the GNU Lesser General Public License, as specified in the COPYING file. -------------------------------------------------------------------------------*/ - -//this is just a compatibility header for jstreams -#inclu... [truncated message content] |
From: <ust...@us...> - 2008-10-02 14:29:03
|
Revision: 2912 http://clucene.svn.sourceforge.net/clucene/?rev=2912&view=rev Author: ustramooner Date: 2008-10-02 14:28:52 +0000 (Thu, 02 Oct 2008) Log Message: ----------- cleanup... added cl_min3 for various new functions cleaned up lucene_utf8towc and lucene_utf8charlen function Modified Paths: -------------- branches/lucene2_3_2/src/core/CLucene/store/IndexInput.cpp branches/lucene2_3_2/src/demo/Main.cpp branches/lucene2_3_2/src/shared/CLucene/_SharedHeader.h branches/lucene2_3_2/src/shared/CLucene/config/gunichartables.cpp branches/lucene2_3_2/src/shared/CLucene/config/repl_wchar.h branches/lucene2_3_2/src/shared/CLucene/config/utf8.cpp branches/lucene2_3_2/src/test/testall.cpp Removed Paths: ------------- branches/lucene2_3_2/src/core/CLucene/store/_IndexInput.h Modified: branches/lucene2_3_2/src/core/CLucene/store/IndexInput.cpp =================================================================== --- branches/lucene2_3_2/src/core/CLucene/store/IndexInput.cpp 2008-10-02 14:21:51 UTC (rev 2911) +++ branches/lucene2_3_2/src/core/CLucene/store/IndexInput.cpp 2008-10-02 14:28:52 UTC (rev 2912) @@ -241,23 +241,5 @@ } -IndexInputStream::IndexInputStream(IndexInput* input){ - this->input = input; - this->size = input->length(); - this->position = input->getFilePointer(); -} -IndexInputStream::~IndexInputStream(){ -} -int32_t IndexInputStream::fillBuffer(char* start, int32_t space){ - int64_t avail = input->length()-input->getFilePointer(); - if ( avail == 0 ) - return -1; - else if ( avail<space ) - space = (int32_t)avail; - - input->readBytes((uint8_t*)start,space); - return space; -} - CL_NS_END Deleted: branches/lucene2_3_2/src/core/CLucene/store/_IndexInput.h =================================================================== --- branches/lucene2_3_2/src/core/CLucene/store/_IndexInput.h 2008-10-02 14:21:51 UTC (rev 2911) +++ branches/lucene2_3_2/src/core/CLucene/store/_IndexInput.h 2008-10-02 14:28:52 UTC (rev 2912) @@ -1,31 +0,0 @@ -/*------------------------------------------------------------------------------ -* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team -* -* Distributable under the terms of either the Apache License (Version 2.0) or -* the GNU Lesser General Public License, as specified in the COPYING file. -------------------------------------------------------------------------------*/ -#ifndef _lucene_store_Internal_IndexInput_ -#define _lucene_store_Internal_IndexInput_ - -#include "CLucene/LuceneThreads.h" -#include "CLucene/util/_bufferedstream.h" - -CL_NS_DEF(store) - - /** - * JStream InputStream which reads from an IndexInput. This class is - * used by the FieldReader to create binary fields. You can then use - * a GZipInputStream to read compressed data or any of the other - * JStream stream types. - * todo: should we really use Buffered? isn't the underlying stream buffered enough? - */ - class IndexInputStream: public jstreams::BufferedInputStream{ - IndexInput* input; - public: - IndexInputStream(IndexInput* input); - ~IndexInputStream(); - int32_t fillBuffer(_stg_byte_t* start, int32_t space); - }; - -CL_NS_END -#endif Modified: branches/lucene2_3_2/src/demo/Main.cpp =================================================================== --- branches/lucene2_3_2/src/demo/Main.cpp 2008-10-02 14:21:51 UTC (rev 2911) +++ branches/lucene2_3_2/src/demo/Main.cpp 2008-10-02 14:28:52 UTC (rev 2912) @@ -11,7 +11,7 @@ //test for memory leaks: #ifdef _MSC_VER #ifdef _DEBUG - #define CRTDBG_MAP_ALLOC + #define _CRTDBG_MAP_ALLOC #include <stdlib.h> #include <crtdbg.h> #endif @@ -29,13 +29,11 @@ int main( int32_t argc, char** argv ){ //Dumper Debug - #ifdef TR_LEAKS - #ifdef _CLCOMPILER_MSVC + #ifdef _MSC_VER #ifdef _DEBUG _CrtSetDbgFlag ( _CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF );//| _CRTDBG_CHECK_ALWAYS_DF | _CRTDBG_CHECK_CRT_DF ); #endif #endif - #endif uint64_t str = Misc::currentTimeMillis(); try{ @@ -63,21 +61,12 @@ _lucene_shutdown(); //clears all static memory //print lucenebase debug -#ifdef LUCENE_ENABLE_MEMLEAKTRACKING - lucene::debug::LuceneBase::__cl_PrintUnclosedObjects(); - //clear memtracking memory (not the unclosed objects) - lucene::debug::LuceneBase::__cl_ClearMemory(); -#endif + //Debuggin techniques: //For msvc, use this for breaking on memory leaks: // _crtBreakAlloc - //to break at this clucene item: - // _lucene_counter_break - //run a memory check before deleting objects: - // _lucene_run_objectcheck - //if LUCENE_ENABLE_CONSTRUCTOR_LOG is on, dont do log if this is true: - // _lucene_disable_debuglogging + //for linux, use valgrind printf ("\n\nTime taken: %d\n\n",Misc::currentTimeMillis() - str); return 0; Modified: branches/lucene2_3_2/src/shared/CLucene/_SharedHeader.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/_SharedHeader.h 2008-10-02 14:21:51 UTC (rev 2911) +++ branches/lucene2_3_2/src/shared/CLucene/_SharedHeader.h 2008-10-02 14:28:52 UTC (rev 2912) @@ -22,8 +22,10 @@ #include "CLucene/config/repl_tchar.h" #include "CLucene/config/repl_wchar.h" -#define cl_min(a,b) (a>b?b:a) -#define cl_max(a,b) (a>b?a:b) +#define cl_min(a,b) ((a)>(b) ? (b) : (a)) +#define cl_min3(a,b,c) ((a)<(b) ? ((a)<(c) ? (a) : (c)) : ((b)<(c) ? (b) : (c))) +#define cl_max(a,b) ((a)>(b) ? (a): (b)) +#define cl_max3(a,b,c) ((a)>(b) ? ((a)>(c) ? (a) : (c)) : ((b)>(c) ? (b) : (c))) #ifdef _CL_HAVE_SAFE_CRT #define cl_sprintf sprintf_s Modified: branches/lucene2_3_2/src/shared/CLucene/config/gunichartables.cpp =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/config/gunichartables.cpp 2008-10-02 14:21:51 UTC (rev 2911) +++ branches/lucene2_3_2/src/shared/CLucene/config/gunichartables.cpp 2008-10-02 14:28:52 UTC (rev 2912) @@ -232,7 +232,7 @@ { const gchar *p = special_case_table + val - 0x1000000; wchar_t ret=0; - lucene_utf8towc(&ret,p,6); + lucene_utf8towc(ret,p); #ifdef _UCS2 return ret; #else @@ -273,7 +273,7 @@ const gchar *p = special_case_table + val - 0x1000000; wchar_t ret=0; - lucene_utf8towc(&ret,p,6); + lucene_utf8towc(ret,p); #ifdef _UCS2 return ret; #else @@ -330,7 +330,7 @@ if (ch == casefold_table[half].ch) { wchar_t ret=0; - lucene_utf8towc(&ret,casefold_table[half].data,6); + lucene_utf8towc(ret,casefold_table[half].data); #ifdef _UCS2 return ret; Modified: branches/lucene2_3_2/src/shared/CLucene/config/repl_wchar.h =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/config/repl_wchar.h 2008-10-02 14:21:51 UTC (rev 2911) +++ branches/lucene2_3_2/src/shared/CLucene/config/repl_wchar.h 2008-10-02 14:28:52 UTC (rev 2912) @@ -20,11 +20,11 @@ CLUCENE_SHARED_EXPORT TCHAR* cl_tcscasefold( TCHAR * str, int len=-1 ); //we provide utf8 conversion functions -CLUCENE_SHARED_EXPORT size_t lucene_utf8towc (wchar_t *ret, const char *s, size_t n); +CLUCENE_SHARED_EXPORT size_t lucene_utf8towc (wchar_t& ret, const char *s); CLUCENE_SHARED_EXPORT size_t lucene_utf8towcs(wchar_t *, const char *, size_t maxslen); CLUCENE_SHARED_EXPORT size_t lucene_wctoutf8 (char * ret, const wchar_t str); CLUCENE_SHARED_EXPORT size_t lucene_wcstoutf8 (char *, const wchar_t *, size_t maxslen); -CLUCENE_SHARED_EXPORT size_t lucene_utf8charlen(const char *p); +CLUCENE_SHARED_EXPORT size_t lucene_utf8charlen(const unsigned char p); //< the number of characters that this first utf8 character will expect //string function replacements #if defined(LUCENE_USE_INTERNAL_CHAR_FUNCTIONS) || (defined(_UCS2) && !defined(_CL_HAVE_FUNCTION_WCSCASECMP)) || (defined(_ASCII) && !defined(_CL_HAVE_FUNCTION_STRCASECMP)) Modified: branches/lucene2_3_2/src/shared/CLucene/config/utf8.cpp =================================================================== --- branches/lucene2_3_2/src/shared/CLucene/config/utf8.cpp 2008-10-02 14:21:51 UTC (rev 2911) +++ branches/lucene2_3_2/src/shared/CLucene/config/utf8.cpp 2008-10-02 14:28:52 UTC (rev 2912) @@ -173,9 +173,9 @@ * valid Unicode characters, you should use lucene_utf8towc_validated() * instead. * - * Return value: the resulting character + * Return value: the number of p consumed for the character **/ -size_t lucene_utf8towc(wchar_t *pwc, const char *p, size_t n) +size_t lucene_utf8towc(wchar_t& pwc, const char *p) { int i, mask = 0; int result; @@ -187,7 +187,7 @@ return 0; UTF8_GET (result, p, i, mask, len); - *pwc = result; + pwc = result; return len; } @@ -210,7 +210,7 @@ wchar_t *rp = result; while (rp < result + result_length && *sp!=0){ - size_t r = lucene_utf8towc(rp,sp,6); + size_t r = lucene_utf8towc(*rp,sp); if ( r == -1 ) return 0; sp += r; @@ -225,11 +225,10 @@ } //get the number of bytes that make up the utf8 character. //this function was not taken from gnome -size_t lucene_utf8charlen(const char *p) +size_t lucene_utf8charlen(const unsigned char c) { int mask = 0; int len=0; - unsigned char c = (unsigned char) *p; UTF8_COMPUTE (c, mask, len); return len; Modified: branches/lucene2_3_2/src/test/testall.cpp =================================================================== --- branches/lucene2_3_2/src/test/testall.cpp 2008-10-02 14:21:51 UTC (rev 2911) +++ branches/lucene2_3_2/src/test/testall.cpp 2008-10-02 14:28:52 UTC (rev 2912) @@ -4,17 +4,17 @@ * Distributable under the terms of either the Apache License (Version 2.0) or * the GNU Lesser General Public License, as specified in the COPYING file. ------------------------------------------------------------------------------*/ -#include "test.h" - -//test for memory leaks: +//msvc test for memory leaks: #ifdef _MSC_VER -#ifdef _DEBUG - #define CRTDBG_MAP_ALLOC - #include <stdlib.h> - #include <crtdbg.h> -#endif -#endif + #ifdef _DEBUG + #define _CRTDBG_MAP_ALLOC + #include <stdlib.h> + #include <crtdbg.h> + #endif +#endif +#include "test.h" + #include <fcntl.h> #ifdef _CL_HAVE_DIRECT_H #include <direct.h> @@ -32,9 +32,10 @@ int main(int argc, char *argv[]) { - #ifdef _CLCOMPILER_MSVC + #ifdef _MSC_VER #ifdef _DEBUG - _CrtSetDbgFlag ( _CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF );// | _CRTDBG_CHECK_ALWAYS_DF | _CRTDBG_CHECK_CRT_DF ); + _CrtSetDbgFlag ( _CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF ); //| _CRTDBG_CHECK_ALWAYS_DF | _CRTDBG_CHECK_CRT_DF ); + _crtBreakAlloc=-1; #endif #endif int ret_result = 0; @@ -202,12 +203,6 @@ _lucene_shutdown(); //clears all static memory //print lucenebase debug -#ifdef LUCENE_ENABLE_MEMLEAKTRACKING - CL_NS(debug)::LuceneBase::__cl_PrintUnclosedObjects(); - //clear memtracking memory (not the unclosed objects) - CL_NS(debug)::LuceneBase::__cl_ClearMemory(); -#endif - if ( ret_result != 0 ) return ret_result; @@ -217,11 +212,6 @@ //Debuggin techniques: //For msvc, use this for breaking on memory leaks: // _crtBreakAlloc - //to break at this clucene item: - // _lucene_counter_break - //run a memory check before deleting objects: - // _lucene_run_objectcheck - //if LUCENE_ENABLE_CONSTRUCTOR_LOG is on, dont do log if this is true: - // _lucene_disable_debuglogging + //for linux, use valgrind } This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |