Menu
▾
▴
clucene-cvs — CLucene SVN Commit log
You can subscribe to this list here.
| 2004 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
(25) |
Dec
(67) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2005 |
Jan
(125) |
Feb
(23) |
Mar
(167) |
Apr
(2) |
May
|
Jun
(19) |
Jul
(304) |
Aug
(181) |
Sep
(189) |
Oct
(145) |
Nov
(110) |
Dec
(44) |
| 2006 |
Jan
(303) |
Feb
(40) |
Mar
(2) |
Apr
(143) |
May
|
Jun
(74) |
Jul
(31) |
Aug
(7) |
Sep
(21) |
Oct
(33) |
Nov
(102) |
Dec
(36) |
| 2007 |
Jan
|
Feb
(16) |
Mar
(38) |
Apr
(34) |
May
(3) |
Jun
(4) |
Jul
(4) |
Aug
(13) |
Sep
(5) |
Oct
|
Nov
|
Dec
|
| 2008 |
Jan
(2) |
Feb
|
Mar
(13) |
Apr
|
May
(18) |
Jun
(48) |
Jul
(136) |
Aug
(45) |
Sep
(21) |
Oct
(32) |
Nov
|
Dec
(9) |
| 2009 |
Jan
(4) |
Feb
|
Mar
(33) |
Apr
(23) |
May
(6) |
Jun
(3) |
Jul
(11) |
Aug
|
Sep
(5) |
Oct
|
Nov
|
Dec
|
| 2010 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
(3) |
Sep
|
Oct
|
Nov
|
Dec
|
|
From: <syn...@us...> - 2009-04-06 12:51:48
|
Revision: 2981
http://clucene.svn.sourceforge.net/clucene/?rev=2981&view=rev
Author: synhershko
Date: 2009-04-06 12:51:42 +0000 (Mon, 06 Apr 2009)
Log Message:
-----------
Fix exception of MFQP test failure.
Tests for MultiFieldQueryParser are still failing, but it's actually is working as expected. The reason for failure in tests is output of toString which does not conform with JLucene. Will be fixed soon.
Modified Paths:
--------------
branches/lucene2_3_2/src/test/queryParser/TestMultiFieldQueryParser.cpp
Modified: branches/lucene2_3_2/src/test/queryParser/TestMultiFieldQueryParser.cpp
===================================================================
--- branches/lucene2_3_2/src/test/queryParser/TestMultiFieldQueryParser.cpp 2009-03-30 21:56:15 UTC (rev 2980)
+++ branches/lucene2_3_2/src/test/queryParser/TestMultiFieldQueryParser.cpp 2009-04-06 12:51:42 UTC (rev 2981)
@@ -63,11 +63,14 @@
const TCHAR* s = q->toString();
int ret = _tcscmp(s,result);
- _CLDELETE_LCARRAY(s);
_CLDELETE(q);
if ( ret != 0 ) {
- CuFail(tc, _T("FAILED Query yielded /%s/, expecting /%s/\n"), s, result);
+ TCHAR buf[HUGE_STRING_LEN];
+ _sntprintf(buf, HUGE_STRING_LEN, _T("FAILED Query yielded /%s/, expecting /%s/\n"), s, result);
+ _CLDELETE_LCARRAY(s);
+ CuFail(tc, buf);
}
+ _CLDELETE_LCARRAY(s);
}
// verify parsing of query using a stopping analyzer
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-30 21:56:27
|
Revision: 2980
http://clucene.svn.sourceforge.net/clucene/?rev=2980&view=rev
Author: ustramooner
Date: 2009-03-30 21:56:15 +0000 (Mon, 30 Mar 2009)
Log Message:
-----------
make only pedantic tests pedantic
Modified Paths:
--------------
branches/lucene2_3_2/src/test/CMakeLists.txt
Modified: branches/lucene2_3_2/src/test/CMakeLists.txt
===================================================================
--- branches/lucene2_3_2/src/test/CMakeLists.txt 2009-03-30 21:46:25 UTC (rev 2979)
+++ branches/lucene2_3_2/src/test/CMakeLists.txt 2009-03-30 21:56:15 UTC (rev 2980)
@@ -79,21 +79,7 @@
SET(TESTS_CXX_FLAGS "${TESTS_CXX_FLAGS} -Wall")
SET(TESTS_EXE_LINKER_FLAGS "${TESTS_EXE_LINKER_FLAGS} -Wall")
ENDIF ( GccFlagWall )
-IF ( GccFlagPedantic )
- SET(TESTS_CXX_FLAGS "${TESTS_CXX_FLAGS} -pedantic")
- SET(TESTS_EXE_LINKER_FLAGS "${TESTS_EXE_LINKER_FLAGS} -pedantic")
-ENDIF ( GccFlagPedantic )
-ADD_EXECUTABLE(cl_test-pedantic EXCLUDE_FROM_ALL ${test_monolithic_Files})
-SET_TARGET_PROPERTIES(cl_test-pedantic PROPERTIES
- COMPILE_DEFINITIONS ""
- COMPILE_FLAGS "${TESTS_CXX_FLAGS} -DMAKE_CLUCENE_SHARED_LIB -DMAKE_CLUCENE_CORE_LIB"
- LINK_FLAGS "${TESTS_EXE_LINKER_FLAGS}")
-ADD_CUSTOM_TARGET(test-pedantic
- COMMENT "Running cl_test-pedantic"
- COMMAND ${EXECUTABLE_OUTPUT_PATH}/cl_test-pedantic
- DEPENDS cl_test-pedantic)
-
ADD_EXECUTABLE(cl_test-ascii EXCLUDE_FROM_ALL ${test_monolithic_Files} )
SET_TARGET_PROPERTIES(cl_test-ascii PROPERTIES
COMPILE_DEFINITIONS "_ASCII"
@@ -156,6 +142,22 @@
COMMAND ${EXECUTABLE_OUTPUT_PATH}/cl_test-platform-charfuncs
DEPENDS cl_test-platform-charfuncs)
+#this one last so we only have pedantic for test-pedantic
+IF ( GccFlagPedantic )
+ SET(TESTS_CXX_FLAGS "${TESTS_CXX_FLAGS} -pedantic")
+ SET(TESTS_EXE_LINKER_FLAGS "${TESTS_EXE_LINKER_FLAGS} -pedantic")
+ENDIF ( GccFlagPedantic )
+ADD_EXECUTABLE(cl_test-pedantic EXCLUDE_FROM_ALL ${test_monolithic_Files})
+SET_TARGET_PROPERTIES(cl_test-pedantic PROPERTIES
+ COMPILE_DEFINITIONS ""
+ COMPILE_FLAGS "${TESTS_CXX_FLAGS} -DMAKE_CLUCENE_SHARED_LIB -DMAKE_CLUCENE_CORE_LIB"
+ LINK_FLAGS "${TESTS_EXE_LINKER_FLAGS}")
+ADD_CUSTOM_TARGET(test-pedantic
+ COMMENT "Running cl_test-pedantic"
+ COMMAND ${EXECUTABLE_OUTPUT_PATH}/cl_test-pedantic
+ DEPENDS cl_test-pedantic)
+
+
#target for running all tests
ADD_CUSTOM_TARGET(test-all
COMMAND ${EXECUTABLE_OUTPUT_PATH}/cl_test-pedantic
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-30 21:46:29
|
Revision: 2979
http://clucene.svn.sourceforge.net/clucene/?rev=2979&view=rev
Author: ustramooner
Date: 2009-03-30 21:46:25 +0000 (Mon, 30 Mar 2009)
Log Message:
-----------
option to build with contribs
Modified Paths:
--------------
branches/lucene2_3_2/CMakeLists.txt
Modified: branches/lucene2_3_2/CMakeLists.txt
===================================================================
--- branches/lucene2_3_2/CMakeLists.txt 2009-03-30 21:44:42 UTC (rev 2978)
+++ branches/lucene2_3_2/CMakeLists.txt 2009-03-30 21:46:25 UTC (rev 2979)
@@ -62,11 +62,16 @@
OPTION(BUILD_STATIC_LIBRARIES
"create targets for building static libraries"
OFF)
+OPTION(BUILD_CONTRIBS
+ "create targets for building the clucene-contribs"
+ OFF)
+OPTION(BUILD_CONTRIBS_LIB
+ "create targets for building the clucene-contribs-lib"
+ OFF)
+SET(LUCENE_SYS_INCLUDES "" CACHE PATH
+ "location for non-system independent files. defaults to CMAKE_INSTALL_PREFIX. see INSTALL documentation for further information."
+ )
-OPTION(LUCENE_SYS_INCLUDES
- "location for non-system independent files. defaults to CMAKE_INSTALL_PREFIX. see INSTALL documentation for further information."
- "")
-
SET ( ENABLE_COMPILE_TESTS_VALUE ON )
IF ( MSVC_IDE )
@@ -132,7 +137,12 @@
ADD_SUBDIRECTORY (src/core)
ADD_SUBDIRECTORY (src/test)
ADD_SUBDIRECTORY (src/demo EXCLUDE_FROM_ALL)
-ADD_SUBDIRECTORY (src/contribs-lib EXCLUDE_FROM_ALL)
+IF ( BUILD_CONTRIBS )
+ ADD_SUBDIRECTORY (src/contribs EXCLUDE_FROM_ALL)
+ENDIF ( BUILD_CONTRIBS )
+IF ( BUILD_CONTRIBS_LIB )
+ ADD_SUBDIRECTORY (src/contribs-lib EXCLUDE_FROM_ALL)
+ENDIF ( BUILD_CONTRIBS_LIB )
#add uninstall command
CONFIGURE_FILE(
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-30 21:44:53
|
Revision: 2978
http://clucene.svn.sourceforge.net/clucene/?rev=2978&view=rev
Author: ustramooner
Date: 2009-03-30 21:44:42 +0000 (Mon, 30 Mar 2009)
Log Message:
-----------
fix EOLs
Modified Paths:
--------------
branches/lucene2_3_2/src/contribs/benchmarker/Benchmarker.cpp
branches/lucene2_3_2/src/contribs/benchmarker/Timer.h
branches/lucene2_3_2/src/contribs/benchmarker/Unit.cpp
branches/lucene2_3_2/src/contribs/benchmarker/Unit.h
Modified: branches/lucene2_3_2/src/contribs/benchmarker/Benchmarker.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/Benchmarker.cpp 2009-03-30 21:38:58 UTC (rev 2977)
+++ branches/lucene2_3_2/src/contribs/benchmarker/Benchmarker.cpp 2009-03-30 21:44:42 UTC (rev 2978)
@@ -30,4 +30,4 @@
timerTotal.stop();
return testsCountSuccess > 0;
-}
\ No newline at end of file
+}
Modified: branches/lucene2_3_2/src/contribs/benchmarker/Timer.h
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/Timer.h 2009-03-30 21:38:58 UTC (rev 2977)
+++ branches/lucene2_3_2/src/contribs/benchmarker/Timer.h 2009-03-30 21:44:42 UTC (rev 2978)
@@ -39,4 +39,4 @@
typedef int (*PTEST_ROUTINE)(Timer*);
-typedef PTEST_ROUTINE LPTEST_ROUTINE;
\ No newline at end of file
+typedef PTEST_ROUTINE LPTEST_ROUTINE;
Modified: branches/lucene2_3_2/src/contribs/benchmarker/Unit.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/Unit.cpp 2009-03-30 21:38:58 UTC (rev 2977)
+++ branches/lucene2_3_2/src/contribs/benchmarker/Unit.cpp 2009-03-30 21:44:42 UTC (rev 2978)
@@ -85,4 +85,4 @@
printf(" avg:%0.3f milliseconds",avg);
}
printf("\n");
-}
\ No newline at end of file
+}
Modified: branches/lucene2_3_2/src/contribs/benchmarker/Unit.h
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/Unit.h 2009-03-30 21:38:58 UTC (rev 2977)
+++ branches/lucene2_3_2/src/contribs/benchmarker/Unit.h 2009-03-30 21:44:42 UTC (rev 2978)
@@ -20,4 +20,4 @@
void runTest(const char* testName,LPTEST_ROUTINE func, int iterations);
virtual void runTests()=0;
-};
\ No newline at end of file
+};
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-30 21:39:15
|
Revision: 2977
http://clucene.svn.sourceforge.net/clucene/?rev=2977&view=rev
Author: ustramooner
Date: 2009-03-30 21:38:58 +0000 (Mon, 30 Mar 2009)
Log Message:
-----------
various contrib code...
Added Paths:
-----------
branches/lucene2_3_2/src/contribs/
branches/lucene2_3_2/src/contribs/CMakeLists.txt
branches/lucene2_3_2/src/contribs/bashscripts/
branches/lucene2_3_2/src/contribs/bashscripts/findPatchThatBrokeUnitTest.sh
branches/lucene2_3_2/src/contribs/bashscripts/simpleupdate.sh
branches/lucene2_3_2/src/contribs/bashscripts/twofileupdate.sh
branches/lucene2_3_2/src/contribs/benchmarker/
branches/lucene2_3_2/src/contribs/benchmarker/Benchmarker.cpp
branches/lucene2_3_2/src/contribs/benchmarker/Benchmarker.h
branches/lucene2_3_2/src/contribs/benchmarker/CMakeLists.txt
branches/lucene2_3_2/src/contribs/benchmarker/Main.cpp
branches/lucene2_3_2/src/contribs/benchmarker/TestCLString.cpp
branches/lucene2_3_2/src/contribs/benchmarker/TestCLString.h
branches/lucene2_3_2/src/contribs/benchmarker/Timer.h
branches/lucene2_3_2/src/contribs/benchmarker/Unit.cpp
branches/lucene2_3_2/src/contribs/benchmarker/Unit.h
branches/lucene2_3_2/src/contribs/benchmarker/stdafx.cpp
branches/lucene2_3_2/src/contribs/benchmarker/stdafx.h
branches/lucene2_3_2/src/contribs/contribs-lib-test/
branches/lucene2_3_2/src/contribs/contribs-lib-test/CMakeLists.txt
branches/lucene2_3_2/src/contribs/contribs-lib-test/CuTest.cpp
branches/lucene2_3_2/src/contribs/contribs-lib-test/CuTest.h
branches/lucene2_3_2/src/contribs/contribs-lib-test/TestAnalysis.cpp
branches/lucene2_3_2/src/contribs/contribs-lib-test/TestHighlight.cpp
branches/lucene2_3_2/src/contribs/contribs-lib-test/TestSnowball.cpp
branches/lucene2_3_2/src/contribs/contribs-lib-test/TestStreams.cpp
branches/lucene2_3_2/src/contribs/contribs-lib-test/TestUtf8.cpp
branches/lucene2_3_2/src/contribs/contribs-lib-test/contribTests.cpp
branches/lucene2_3_2/src/contribs/contribs-lib-test/test.h
branches/lucene2_3_2/src/contribs/contribs-lib-test/testall.cpp
Added: branches/lucene2_3_2/src/contribs/CMakeLists.txt
===================================================================
--- branches/lucene2_3_2/src/contribs/CMakeLists.txt (rev 0)
+++ branches/lucene2_3_2/src/contribs/CMakeLists.txt 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,2 @@
+ADD_SUBDIRECTORY (contribs-lib-test EXCLUDE_FROM_ALL)
+ADD_SUBDIRECTORY (benchmarker EXCLUDE_FROM_ALL)
Added: branches/lucene2_3_2/src/contribs/bashscripts/findPatchThatBrokeUnitTest.sh
===================================================================
--- branches/lucene2_3_2/src/contribs/bashscripts/findPatchThatBrokeUnitTest.sh (rev 0)
+++ branches/lucene2_3_2/src/contribs/bashscripts/findPatchThatBrokeUnitTest.sh 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,113 @@
+#! /bin/bash
+#
+# (c) Jos van den Oever <jo...@va...>
+# modified by (c) Ben van Klinken <ust...@us...
+#
+# This script checks out clucen and runs the given unit test. If the unit test
+# fails, it goes back to the previous version, compiles and tests again.
+# This goes on until the unit test is not present or runs successfully.
+
+# check the arguments
+if (( $# != 1 && $# != 2)); then
+ echo Usage: $0 testname
+ echo Note: This script must be run from the base
+ exit
+fi
+
+# the path to the unit test executable
+TESTNAME=$1
+
+## Configuration parameters
+
+# the maximal number of log entries to download
+MAXLOGENTRIES=300
+
+# the maximal number of steps you wish to take
+MAXSTEPS=300
+
+# make exectable with arguments
+MAKE="make"
+
+# should we do a drastic cleanup between runs or not?
+FORCEFULLBUILD=0
+
+#get the source dir
+SOURCEDIR=`grep CMAKE_HOME_DIRECTORY CMakeCache.txt |perl -pi -e 's/.*=(.*)/\1/'`
+if [ "$SOURCEDIR" == "" ]; then
+ echo "Run cmake before running this script"
+ exit 1
+fi
+
+####################
+
+# function for testing a particular test in a particular revision
+function runTest {
+ REVISION=$1
+ echo Testing revision $REVISION.
+
+
+ exit 0
+
+ # go back to the given revision
+ cd $TESTDIR
+ svn update $MODULE -r $REVISION $SOURCEDIR
+ if (( $? != 0 )); then
+ # if updating failed, we have to get a fresh version
+ rm -rf $TESTDIR/$MODULE
+ svn checkout -r $REVISION $SVNURL $SOURCEDIR
+ if (( $? != 0 )); then exit; fi
+ fi
+
+ # configure the code
+ # if we cannot configure the test, we continue to the next revision number
+ if (( $FORCEFULLBUILD == 1 )); then
+ rm -rf $TESTDIR/$MODULE/build
+ fi
+ mkdir $TESTDIR/$MODULE/build
+ cd $TESTDIR/$MODULE/build
+ cmake $SOURCEDIR
+ if (( $? != 0 )); then return; fi
+
+ # get the name of the unit test and build it
+ # if we cannot build the test, we continue to the next revision number
+ echo $MAKE $TESTNAME
+ $MAKE $TESTNAME
+ if (( $? != 0 )); then return; fi
+
+ # find the test executable
+ TESTPATH=`find -name $TESTNAME -type f -perm -u+x`
+
+ # run the unit test and exit if it ran without error
+ $TESTPATH
+ if (( $? == 0 )); then
+ echo The last revision where the test $TESTNAME worked was $REVISION.
+ BROKEN=`grep -B 1 $REVISION $TESTDIR/revisions |head -1`
+ echo The first revision that was broken was $BROKEN:
+ svn log -r $BROKEN $TESTDIR/$MODULE $SOURCEDIR
+ exit
+ fi
+}
+
+# determine the URL of the svn repository
+SVNURL=`svn info $SOURCEDIR | grep -m 1 '^URL: ' | cut -b 6-`
+if (( $? != 0 )); then exit; fi
+
+echo $SVNURL
+
+# determine the module name
+MODULE=`basename $SVNURL`
+
+echo $MODULE
+
+# get the last 100 relevant version numbers
+svn log $MODULE --limit $MAXLOGENTRIES --non-interactive $SOURCEDIR\
+ | grep -E '^r[0123456789]+' \
+ | perl -pi -e 's/^r(\d+).*/\1/' | head -n $MAXSTEPS > revisions
+if (( $? != 0 )); then exit; fi
+
+for REVISION in `cat revisions`; do
+ runTest $REVISION;
+done
+
+echo No revision was found in which the unit test worked.
+
Added: branches/lucene2_3_2/src/contribs/bashscripts/simpleupdate.sh
===================================================================
--- branches/lucene2_3_2/src/contribs/bashscripts/simpleupdate.sh (rev 0)
+++ branches/lucene2_3_2/src/contribs/bashscripts/simpleupdate.sh 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,41 @@
+#! /bin/bash
+
+# this test checks if the strigicmd utility properly detect creation and
+# deletion of a file
+
+function fail() {
+ echo Test failed
+ exit 1
+}
+
+STRIGICMD="`find -type f -name strigicmd -type f -perm -o+x` "
+echo Using $STRIGICMD
+NTHREADS=1
+
+rm -r x y 2> /dev/null
+mkdir x
+touch x/y
+touch x/z
+echo == $STRIGICMD create -t clucene -d y x ==
+if ! $STRIGICMD create -j 1 -t clucene -d y x; then
+ fail
+fi
+echo == $STRIGICMD listFiles -t clucene -d y ==
+if ! $STRIGICMD listFiles -t clucene -d y; then
+ fail
+fi
+rm x/y
+echo == $STRIGICMD update -j $NTHREADS -t clucene -d y x ==
+if ! $STRIGICMD update -j $NTHREADS -t clucene -d y x; then
+ fail
+fi
+echo == $STRIGICMD listFiles -t clucene -d y ==
+if ! $STRIGICMD listFiles -t clucene -d y; then
+ fail
+fi
+OUT=`$STRIGICMD listFiles -t clucene -d y`
+if [[ $OUT == $'x\nx/z' ]]; then
+ echo Test succesfull
+ exit 0
+fi
+fail
Added: branches/lucene2_3_2/src/contribs/bashscripts/twofileupdate.sh
===================================================================
--- branches/lucene2_3_2/src/contribs/bashscripts/twofileupdate.sh (rev 0)
+++ branches/lucene2_3_2/src/contribs/bashscripts/twofileupdate.sh 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,42 @@
+#! /bin/bash
+
+# this test checks if the strigicmd utility properly detect creation and
+# deletion of a file
+
+function fail() {
+ echo Test failed
+ exit 1
+}
+
+#VG="valgrind --db-attach=yes "
+
+rm -r x y
+mkdir x
+touch x/y
+touch x/z
+echo == src/strigicmd/strigicmd create -t clucene -d y x ==
+if ! $VG src/strigicmd/strigicmd create -t clucene -d y x; then
+ fail
+fi
+echo == src/strigicmd/strigicmd listFiles -t clucene -d y ==
+if ! $VG src/strigicmd/strigicmd listFiles -t clucene -d y; then
+ fail
+fi
+sleep 1
+touch x/y
+touch x/z
+echo == src/strigicmd/strigicmd update -t clucene -d y x ==
+exit
+if ! $VG src/strigicmd/strigicmd update -t clucene -d y x; then
+ fail
+fi
+echo == src/strigicmd/strigicmd listFiles -t clucene -d y ==
+if ! $VG src/strigicmd/strigicmd listFiles -t clucene -d y; then
+ fail
+fi
+OUT=`$VG src/strigicmd/strigicmd listFiles -t clucene -d y`
+if [[ $OUT == 'x/z' ]]; then
+ echo Test succesfull
+ exit 0
+fi
+fail
Added: branches/lucene2_3_2/src/contribs/benchmarker/Benchmarker.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/Benchmarker.cpp (rev 0)
+++ branches/lucene2_3_2/src/contribs/benchmarker/Benchmarker.cpp 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,33 @@
+#include "stdafx.h"
+#include "Benchmarker.h"
+#include "Unit.h"
+
+void Benchmarker::Add(Unit* unit){
+ tests.push_back(unit);
+}
+Benchmarker::Benchmarker(void)
+{
+ reset();
+}
+void Benchmarker::reset(){
+ timerTotal.reset();
+ testsCountTotal=0;
+ testsCountSuccess=0;
+ testsRunTotal=0;
+ testsRunSuccess=0;
+}
+bool Benchmarker::run(){
+ timerTotal.start();
+ printf( ">> running tests...\n" );
+ for ( int i=0;i<tests.size();i++ ){
+ Unit* unit = tests[i];
+ unit->start(this);
+ unit->stop();
+ }
+ printf( "\n>> benchmarker ran a total of %d test cases(%d successes) in %d ms\n",
+ testsCountTotal,testsCountSuccess,
+ (int32_t)timerTotal.interval() );
+ timerTotal.stop();
+
+ return testsCountSuccess > 0;
+}
\ No newline at end of file
Added: branches/lucene2_3_2/src/contribs/benchmarker/Benchmarker.h
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/Benchmarker.h (rev 0)
+++ branches/lucene2_3_2/src/contribs/benchmarker/Benchmarker.h 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,17 @@
+#pragma once
+
+class Benchmarker
+{
+ lucene::util::CLVector<Unit*> tests;
+public:
+ Timer timerTotal;
+ int testsCountTotal;
+ int testsCountSuccess;
+ int testsRunTotal;
+ int testsRunSuccess;
+
+ Benchmarker(void);
+ void Add(Unit* unit);
+ bool run();
+ void reset();
+};
Added: branches/lucene2_3_2/src/contribs/benchmarker/CMakeLists.txt
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/CMakeLists.txt (rev 0)
+++ branches/lucene2_3_2/src/contribs/benchmarker/CMakeLists.txt 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,20 @@
+PROJECT(clucene-benchmarker)
+
+INCLUDE (DefineOptions)
+DEFINE_OPTIONS(EXTRA_OPTIONS)
+ADD_DEFINITIONS(${EXTRA_OPTIONS})
+
+file(GLOB_RECURSE benchmarker_HEADERS ${clucene-benchmarker_SOURCE_DIR}/*.h)
+
+SET(benchmarker_files
+ ./Benchmarker.cpp
+ ./Main.cpp
+ ./stdafx.cpp
+ ./Unit.cpp
+
+ ./TestCLString.cpp
+ ${benchmarker_HEADERS}
+)
+
+ADD_EXECUTABLE(cl_benchmarker EXCLUDE_FROM_ALL ${benchmarker_files} )
+TARGET_LINK_LIBRARIES(cl_benchmarker clucene-core clucene-shared)
Added: branches/lucene2_3_2/src/contribs/benchmarker/Main.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/Main.cpp (rev 0)
+++ branches/lucene2_3_2/src/contribs/benchmarker/Main.cpp 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,103 @@
+#include "stdafx.h"
+#include "TestCLString.h"
+
+#ifdef COMPILER_MSVC
+#ifdef _DEBUG
+ #define CRTDBG_MAP_ALLOC
+ #include <stdlib.h>
+ #include <crtdbg.h>
+#endif
+#endif
+
+#include <stdlib.h>
+
+#include <fcntl.h>
+#ifdef _CL_HAVE_DIRECT_H
+ #include <direct.h>
+#endif
+#ifdef _CL_HAVE_SYS_STAT_H
+ #include <sys/stat.h>
+#endif
+#ifdef _CL_HAVE_IO_H
+ #include <io.h>
+#endif
+
+using namespace std;
+using namespace lucene::util;
+
+const char* cl_tempDir;
+char clucene_data_location[1024];
+
+int main( int argc, char** argv ){
+ //Dumper Debug
+ #ifdef COMPILER_MSVC
+ #ifdef _DEBUG
+ _CrtSetDbgFlag ( _CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF );//| _CRTDBG_CHECK_ALWAYS_DF | _CRTDBG_CHECK_CRT_DF );
+ #endif
+ #endif
+
+ Benchmarker bench;
+ TestCLString clstring;
+ bool ret_result = false;
+
+ cl_tempDir = NULL;
+ if ( Misc::dir_Exists("/tmp") )
+ cl_tempDir = "/tmp";
+ if ( getenv("TEMP") != NULL )
+ cl_tempDir = getenv("TEMP");
+ else if ( getenv("TMP") != NULL )
+ cl_tempDir = getenv("TMP");
+
+ char* tmp = _CL_NEWARRAY(char,strlen(cl_tempDir)+9);
+ strcpy(tmp,cl_tempDir);
+ strcat(tmp,"/clucene");
+ _mkdir(tmp);
+ if ( Misc::dir_Exists(tmp) )
+ cl_tempDir=tmp;
+
+
+ clucene_data_location[0]=0;
+ if ( CL_NS(util)::Misc::dir_Exists(CLUCENE_DATA_LOCATION1 "/reuters-21578-index/segments") )
+ strcpy(clucene_data_location, CLUCENE_DATA_LOCATION1);
+ else if ( CL_NS(util)::Misc::dir_Exists(CLUCENE_DATA_LOCATION2 "/reuters-21578-index/segments") )
+ strcpy(clucene_data_location, CLUCENE_DATA_LOCATION2);
+ else if ( CL_NS(util)::Misc::dir_Exists(CLUCENE_DATA_LOCATION3 "/reuters-21578-index/segments") )
+ strcpy(clucene_data_location, CLUCENE_DATA_LOCATION3);
+ else if ( getenv(CLUCENE_DATA_LOCATIONENV) != NULL ){
+ strcpy(clucene_data_location,getenv(CLUCENE_DATA_LOCATIONENV));
+ strcat(clucene_data_location,"/data/reuters-21578-index/segments");
+ if ( CL_NS(util)::Misc::dir_Exists( clucene_data_location ) ){
+ strcpy(clucene_data_location, getenv(CLUCENE_DATA_LOCATIONENV));
+ strcat(clucene_data_location, "/data");
+ }else
+ clucene_data_location[0]=0;
+ }
+
+ /* first check that we are running the test for the correct position */
+ //todo: make this configurable
+ if ( !*clucene_data_location ){
+ fprintf(stderr,"%s must be run from a subdirectory of the application's root directory\n",argv[0]);
+ fprintf(stderr,"ensure that the test data exists in %s or %s or %s\n",CLUCENE_DATA_LOCATION1, CLUCENE_DATA_LOCATION2, CLUCENE_DATA_LOCATION3);
+ if ( getenv(CLUCENE_DATA_LOCATIONENV) != NULL )
+ fprintf(stderr,"%s/data was also checked because of the " CLUCENE_DATA_LOCATIONENV " environment variable", getenv(CLUCENE_DATA_LOCATIONENV));
+ ret_result = 1;
+ goto exit_point;
+ }
+
+
+ bench.Add(&clstring);
+ ret_result = bench.run();
+
+
+
+exit_point:
+ _lucene_shutdown(); //clears all static memory
+ //print lucenebase debug
+
+ return ret_result ? 0 : 1;
+
+ //Debuggin techniques:
+ //For msvc, use this for breaking on memory leaks:
+ // _crtBreakAlloc
+ //for linux, use valgrind
+}
Added: branches/lucene2_3_2/src/contribs/benchmarker/TestCLString.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/TestCLString.cpp (rev 0)
+++ branches/lucene2_3_2/src/contribs/benchmarker/TestCLString.cpp 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,51 @@
+#include "stdafx.h"
+
+using namespace lucene::util;
+using namespace lucene::analysis;
+using namespace lucene::document;
+using namespace lucene::index;
+using namespace lucene::store;
+
+int BenchmarkDocumentWriter(Timer* timerCase)
+{
+ RAMDirectory ram;
+ SimpleAnalyzer an;
+ IndexWriter* ndx = _CLNEW IndexWriter(&ram, &an, true);
+ ndx->setMaxFieldLength(0x7FFFFFFF);
+
+ char fname[1024];
+ strcpy(fname, clucene_data_location);
+ strcat(fname, "reuters-21578/feldman-cia-worldfactbook-data.txt");
+
+ timerCase->start();
+ for ( int i=0;i<10;i++ ){
+
+ FileReader* reader = _CLNEW FileReader(fname, "ASCII");
+ Document doc;
+ doc.add(*_CLNEW Field(_T("contents"),reader, Field::STORE_YES | Field::INDEX_TOKENIZED));
+
+ ndx->addDocument(&doc);
+ }
+ ndx->close();
+ timerCase->stop();
+
+ ram.close();
+ _CLDELETE(ndx);
+ return 0;
+}
+
+int BenchmarkTermDocs(Timer* timerCase){
+ IndexReader* reader = IndexReader::open("index");
+ timerCase->start();
+ TermEnum* en = reader->terms();
+ while (en->next()){
+ Term* term = en->term();
+ _CLDECDELETE(term);
+ }
+ en->close();
+ _CLDELETE(en);
+ timerCase->stop();
+ reader->close();
+ _CLDELETE(reader);
+ return 0;
+}
Added: branches/lucene2_3_2/src/contribs/benchmarker/TestCLString.h
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/TestCLString.h (rev 0)
+++ branches/lucene2_3_2/src/contribs/benchmarker/TestCLString.h 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,17 @@
+#pragma once
+
+int BenchmarkDocumentWriter(Timer*);
+int BenchmarkTermDocs(Timer* timerCase);
+
+class TestCLString:public Unit
+{
+protected:
+ void runTests(){
+ this->runTest("BenchmarkDocumentWriter",BenchmarkDocumentWriter,10);
+ //this->runTest("BenchmarkTermDocs",BenchmarkTermDocs,100);
+ }
+public:
+ const char* getName(){
+ return "TestCLString";
+ }
+};
Added: branches/lucene2_3_2/src/contribs/benchmarker/Timer.h
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/Timer.h (rev 0)
+++ branches/lucene2_3_2/src/contribs/benchmarker/Timer.h 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,42 @@
+#pragma once
+
+class Timer{
+public:
+ int64_t startTime;
+ int64_t stopTime;
+ bool running;
+ Timer(){
+ running=false;
+ reset();
+ }
+ void reset(){
+ startTime=0;
+ stopTime=0;
+ running=false;
+ }
+ void start(){
+ startTime = lucene::util::Misc::currentTimeMillis();
+ running=true;
+ }
+ int32_t split(){
+ return lucene::util::Misc::currentTimeMillis()-startTime;
+ }
+ int32_t stop(){
+ if ( running ){
+ running=false;
+ stopTime = lucene::util::Misc::currentTimeMillis();
+ }
+ return stopTime-startTime;
+ }
+ int32_t interval(){
+ if (running)
+ return lucene::util::Misc::currentTimeMillis()-startTime;
+ else
+ return stopTime-startTime;
+ }
+
+};
+
+
+typedef int (*PTEST_ROUTINE)(Timer*);
+typedef PTEST_ROUTINE LPTEST_ROUTINE;
\ No newline at end of file
Added: branches/lucene2_3_2/src/contribs/benchmarker/Unit.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/Unit.cpp (rev 0)
+++ branches/lucene2_3_2/src/contribs/benchmarker/Unit.cpp 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,88 @@
+#include "stdafx.h"
+#include "Unit.h"
+
+Unit::Unit()
+{
+ this->bm=NULL;
+ testsCountTotal=0;
+ testsCountSuccess=0;
+ testsRunTotal=0;
+ testsRunSuccess=0;
+ timerCase.reset();
+ timerTotal.reset();
+}
+
+void Unit::stop(){
+ timerTotal.stop();
+ bm=NULL;
+
+ printf( "> unit ran a total of %d test cases(%d successes) in %d ms\n",
+ testsCountTotal,testsCountSuccess,
+ (int)timerTotal.interval() );
+}
+
+void Unit::start(Benchmarker* bm){
+ this->bm = bm;
+ timerTotal.start();
+
+ printf( "> running unit %s\n", getName() );
+ runTests();
+}
+void Unit::runTest(const char* testName,LPTEST_ROUTINE func, int iterations){
+ if ( bm == NULL )
+ _CLTHROWA(CL_ERR_NullPointer, "Unit not started with benchmarker!");
+ float avg=0;
+ int32_t min=0;
+ int32_t max=0;
+ int count=0;
+ Timer total;
+ bool success = false;
+
+ try {
+ total.start();
+ printf("\n > running %s %d times...", testName, iterations);
+ for ( int i=0;i<iterations;i++ ){
+ timerCase.reset();
+ success = (func(&timerCase) == 0 );
+ int32_t t = timerCase.stop();
+ if ( count == 0 ){
+ min = t;
+ max = t;
+ avg = t;
+ }else{
+ if ( t < min )
+ min = t;
+ if ( t > max )
+ max = t;
+ avg = (avg + t)/2;
+ }
+
+ testsRunTotal++;
+ bm->testsRunTotal++;
+ if ( success ){
+ testsRunSuccess++;
+ bm->testsRunSuccess++;
+ }
+ count++;
+ }
+ success = true;
+ }catch(CLuceneError& err){
+ printf("\n > error occurred: %s\n", err.what());
+ }catch(...){
+ printf("\n > unexpected error occurred\n >");
+ }
+ testsCountTotal++;
+ bm->testsCountTotal++;
+ if ( success ){
+ testsCountSuccess++;
+ bm->testsCountSuccess++;
+ }
+ printf(" it took %d milliseconds",total.stop());
+
+ if ( iterations > 1 ){
+ printf("\n\tmin:%d",min);
+ printf(" max:%d,",max);
+ printf(" avg:%0.3f milliseconds",avg);
+ }
+ printf("\n");
+}
\ No newline at end of file
Added: branches/lucene2_3_2/src/contribs/benchmarker/Unit.h
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/Unit.h (rev 0)
+++ branches/lucene2_3_2/src/contribs/benchmarker/Unit.h 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,23 @@
+#pragma once
+#include "CLucene/util/Misc.h"
+
+
+class Unit
+{
+public:
+ void start(Benchmarker* benchmarker);
+ void stop();
+ virtual const char* getName()=0;
+ Unit();
+protected:
+ Timer timerCase;
+ Timer timerTotal;
+ int testsCountTotal;
+ int testsCountSuccess;
+ int testsRunTotal;
+ int testsRunSuccess;
+ Benchmarker* bm;
+
+ void runTest(const char* testName,LPTEST_ROUTINE func, int iterations);
+ virtual void runTests()=0;
+};
\ No newline at end of file
Added: branches/lucene2_3_2/src/contribs/benchmarker/stdafx.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/stdafx.cpp (rev 0)
+++ branches/lucene2_3_2/src/contribs/benchmarker/stdafx.cpp 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,5 @@
+// stdafx.cpp : source file that includes just the standard includes
+// demo.pch will be the pre-compiled header
+// stdafx.obj will contain the pre-compiled type information
+
+#include "stdafx.h"
Added: branches/lucene2_3_2/src/contribs/benchmarker/stdafx.h
===================================================================
--- branches/lucene2_3_2/src/contribs/benchmarker/stdafx.h (rev 0)
+++ branches/lucene2_3_2/src/contribs/benchmarker/stdafx.h 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,31 @@
+// stdafx.h : include file for standard system include files,
+// or project specific include files that are used frequently, but
+// are changed infrequently
+//
+#ifndef _lucene_examples_benchmark_stdafx_
+#define _lucene_examples_benchmark_stdafx_
+
+#define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers
+#include <stdio.h>
+//#include <tchar.h>
+#include <string.h>
+
+#include "CLucene.h"
+#include "CLucene/_clucene-config.h"
+#include "CLucene/util/Misc.h"
+#include "CLucene/store/RAMDirectory.h"
+
+#define CLUCENE_DATA_LOCATION1 "../../src/test/data/"
+#define CLUCENE_DATA_LOCATION2 "../src/test/data/"
+#define CLUCENE_DATA_LOCATION3 "../../../src/test/data/"
+#define CLUCENE_DATA_LOCATIONENV "srcdir"
+
+extern const char* cl_tempDir;
+extern char clucene_data_location[1024];
+
+class Benchmarker;
+#include "Timer.h"
+#include "Unit.h"
+#include "Benchmarker.h"
+
+#endif
Added: branches/lucene2_3_2/src/contribs/contribs-lib-test/CMakeLists.txt
===================================================================
--- branches/lucene2_3_2/src/contribs/contribs-lib-test/CMakeLists.txt (rev 0)
+++ branches/lucene2_3_2/src/contribs/contribs-lib-test/CMakeLists.txt 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,33 @@
+PROJECT(clucene-contribs-lib-test)
+
+INCLUDE (DefineOptions)
+DEFINE_OPTIONS(EXTRA_OPTIONS)
+ADD_DEFINITIONS(${EXTRA_OPTIONS})
+
+INCLUDE_DIRECTORIES( ${clucene-contribs-lib-test_SOURCE_DIR} )
+INCLUDE_DIRECTORIES( ${clucene-contribs-lib_SOURCE_DIR} )
+
+file(GLOB_RECURSE test_HEADERS ${CMAKE_SOURCE_DIR}/test/*.h)
+
+SET(test_files
+ ./contribTests.cpp
+ ./TestHighlight.cpp
+ ./TestSnowball.cpp
+ ./TestStreams.cpp
+ ./TestUtf8.cpp
+ ./TestAnalysis.cpp
+ ./CuTest.cpp
+ ./testall.cpp
+ ${test_HEADERS}
+)
+IF ( USE_SHARED_OBJECT_FILES )
+ GET_SHARED_FILES(clucene_shared_Files)
+ENDIF ( USE_SHARED_OBJECT_FILES )
+
+#todo: do glob header and include header files for IDE.
+ADD_EXECUTABLE(cl_contribs-lib-test EXCLUDE_FROM_ALL ${clucene_shared_Files} ${test_files} )
+
+#link the executable against the releavent clucene-shared library (if we aren't using the object files)
+IF ( NOT USE_SHARED_OBJECT_FILES )
+ TARGET_LINK_LIBRARIES(cl_contribs-lib-test clucene-core clucene-shared clucene-contribs-lib)
+ENDIF ( NOT USE_SHARED_OBJECT_FILES )
Added: branches/lucene2_3_2/src/contribs/contribs-lib-test/CuTest.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs/contribs-lib-test/CuTest.cpp (rev 0)
+++ branches/lucene2_3_2/src/contribs/contribs-lib-test/CuTest.cpp 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,536 @@
+/*------------------------------------------------------------------------------
+ * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
+ *
+ * Distributable under the terms of either the Apache License (Version 2.0) or
+ * the GNU Lesser General Public License, as specified in the COPYING file.
+------------------------------------------------------------------------------*/
+#include "test.h"
+#include "CuTest.h"
+#include <stdlib.h>
+#include <assert.h>
+
+static int verbose = 0;
+static int messyPrinting = 0;
+
+void CuInit(int argc, char *argv[]) {
+ int i;
+
+ for (i = 0; i < argc; i++) {
+ if (!strcmp(argv[i], "-v")) {
+ verbose = 1;
+ }
+ if (!strcmp(argv[i], "-p")) {
+ messyPrinting = 1;
+ }
+ }
+}
+
+/*-------------------------------------------------------------------------*
+ * CuTcs
+ *-------------------------------------------------------------------------*/
+TCHAR* CuStrAlloc(int size) {
+ TCHAR* n = (TCHAR*) malloc(sizeof (TCHAR) * (size));
+ return n;
+}
+
+TCHAR* CuTcsCopy(TCHAR* old) {
+ int len = _tcslen(old);
+ TCHAR* n = CuStrAlloc(len + 1);
+ _tcscpy(n, old);
+ return n;
+}
+
+TCHAR* CuTcsCat(TCHAR* orig, TCHAR* add) {
+ int len = _tcslen(orig) + _tcslen(add);
+ TCHAR* n = CuStrAlloc(len + 1);
+ _tcscpy(n, orig);
+ _tcscat(n, add);
+ return n;
+}
+
+/*-------------------------------------------------------------------------*
+ * CuString
+ *-------------------------------------------------------------------------*/
+
+TCHAR* CuTcsAlloc(int size) {
+ TCHAR* n = (TCHAR*) malloc(sizeof (TCHAR) * (size));
+ return n;
+}
+
+TCHAR* CuTcsCopy(const TCHAR* old) {
+ int len = _tcslen(old);
+ TCHAR* n = CuTcsAlloc(len + 1);
+ _tcscpy(n, old);
+ return n;
+}
+
+/*-------------------------------------------------------------------------*
+ * CuString
+ *-------------------------------------------------------------------------*/
+
+void CuStringInit(CuString* str) {
+ str->length = 0;
+ str->size = STRING_MAX;
+ str->buffer = (TCHAR*) malloc(sizeof (TCHAR) * str->size);
+ str->buffer[0] = '\0';
+}
+
+CuString* CuStringNew(void) {
+ CuString* str = (CuString*) malloc(sizeof (CuString));
+ str->length = 0;
+ str->size = STRING_MAX;
+ str->buffer = (TCHAR*) malloc(sizeof (TCHAR) * str->size);
+ str->buffer[0] = '\0';
+ return str;
+}
+
+void CuStringFree(CuString* str) {
+ free(str->buffer);
+ free(str);
+}
+
+void CuStringResize(CuString* str, int newSize) {
+ str->buffer = (TCHAR*) realloc(str->buffer, sizeof (TCHAR) * newSize);
+ str->size = newSize;
+}
+
+void CuStringAppend(CuString* str, const TCHAR* text) {
+ int length = _tcslen(text);
+ if (str->length + length + 1 >= str->size)
+ CuStringResize(str, str->length + length + 1 + STRING_INC);
+ str->length += length;
+ _tcscat(str->buffer, text);
+}
+
+void CuStringAppendChar(CuString* str, TCHAR ch) {
+ TCHAR text[2];
+ text[0] = ch;
+ text[1] = '\0';
+ CuStringAppend(str, text);
+}
+
+void CuStringAppendFormat(CuString* str, const TCHAR* format, ...) {
+ TCHAR buf[HUGE_STRING_LEN];
+ va_list argp;
+ va_start(argp, format);
+ _vsntprintf(buf, HUGE_STRING_LEN, format, argp);
+ va_end(argp);
+ CuStringAppend(str, buf);
+}
+
+void CuStringRead(CuString *str, TCHAR *path) {
+ path = NULL;
+ CU_TDUP(path, str->buffer);
+}
+
+/*-------------------------------------------------------------------------*
+ * CuTest
+ *-------------------------------------------------------------------------*/
+
+void CuTestInit(CuTest* t, const TCHAR* name, TestFunction function) {
+ t->name = CuTcsCopy(name);
+ t->notimpl = 0;
+ t->failed = 0;
+ t->ran = 0;
+ t->message = NULL;
+ t->function = function;
+ // t->jumpBuf = NULL;
+}
+
+CuTest* CuTestNew(const TCHAR* name, TestFunction function) {
+ CuTest* tc = CU_ALLOC(CuTest);
+ CuTestInit(tc, name, function);
+ return tc;
+}
+
+void CuTestDelete(CuTest* tst) {
+ free(tst->name);
+ if (tst->message != NULL)
+ free(tst->message);
+ free(tst);
+}
+
+void CuNotImpl(CuTest* tc, const TCHAR* message) {
+ CuString* newstr = CuStringNew();
+ CuStringAppend(newstr, message);
+ CuStringAppend(newstr, _T(" not implemented on this platform"));
+ tc->notimpl = 1;
+ CuMessage(tc, newstr->buffer);
+ CuStringFree(newstr);
+ // if (tc->jumpBuf != 0) longjmp(*(tc->jumpBuf), 0);
+}
+
+void CuFail(CuTest* tc, const TCHAR* format, ...) {
+ tc->failed = 1;
+
+ TCHAR buf[HUGE_STRING_LEN];
+ va_list argp;
+ va_start(argp, format);
+ _vsntprintf(buf, HUGE_STRING_LEN, format, argp);
+ va_end(argp);
+
+ // CuMessage(tc,buf);
+ _CLTHROWT(CL_ERR_Runtime, buf);
+}
+
+void CuMessageV(CuTest* tc, const TCHAR* format, va_list& argp) {
+ TCHAR buf[HUGE_STRING_LEN];
+ _vsntprintf(buf, HUGE_STRING_LEN, format, argp);
+
+ TCHAR* old = tc->message;
+ if (messyPrinting) {
+ _tprintf(_T("%s"), buf);
+ } else {
+ if (old == NULL) {
+ tc->message = CuTcsCopy(buf);
+ } else {
+ tc->message = CuTcsCat(old, buf);
+ free(old);
+ }
+ }
+}
+
+void CuMessage(CuTest* tc, const TCHAR* format, ...) {
+ va_list argp;
+ va_start(argp, format);
+ CuMessageV(tc, format, argp);
+ va_end(argp);
+}
+
+void CuMessageA(CuTest* tc, const char* format, ...) {
+ va_list argp;
+ char buf[HUGE_STRING_LEN];
+ TCHAR tbuf[HUGE_STRING_LEN];
+ va_start(argp, format);
+ vsprintf(buf, format, argp);
+ va_end(argp);
+
+ TCHAR* old = tc->message;
+ STRCPY_AtoT(tbuf, buf, HUGE_STRING_LEN);
+ if (messyPrinting) {
+ _tprintf(_T("%s"), buf);
+ } else {
+ if (old == NULL) {
+ tc->message = CuTcsCopy(tbuf);
+ } else {
+ tc->message = CuTcsCat(old, tbuf);
+ free(old);
+ }
+ }
+}
+
+void CuAssert(CuTest* tc, const TCHAR* message, int condition) {
+ if (condition) return;
+ CuFail(tc, message);
+}
+
+void CuAssertTrue(CuTest* tc, int condition) {
+ if (condition) return;
+ CuFail(tc, _T("assert failed"));
+}
+
+void CuAssertStrEquals(CuTest* tc, const TCHAR* preMessage, const TCHAR* expected, const TCHAR* actual) {
+ CuString* message;
+ if (_tcscmp(expected, actual) == 0) return;
+ message = CuStringNew();
+ CuStringAppend(message, preMessage);
+ CuStringAppend(message, _T(" : "));
+ CuStringAppend(message, _T("expected\n---->\n"));
+ CuStringAppend(message, expected);
+ CuStringAppend(message, _T("\n<----\nbut saw\n---->\n"));
+ CuStringAppend(message, actual);
+ CuStringAppend(message, _T("\n<----"));
+ CuFail(tc, message->buffer);
+ CuStringFree(message);
+}
+
+void CuAssertIntEquals(CuTest* tc, const TCHAR* preMessage, int expected, int actual) {
+ TCHAR buf[STRING_MAX];
+ if (expected == actual) return;
+ _sntprintf(buf, STRING_MAX, _T("%s : expected <%d> but was <%d>"), preMessage, expected, actual);
+ CuFail(tc, buf);
+}
+
+void CuAssertPtrEquals(CuTest* tc, const TCHAR* preMessage, const void* expected, const void* actual) {
+ TCHAR buf[STRING_MAX];
+ if (expected == actual) return;
+ _sntprintf(buf, STRING_MAX, _T("%s : expected pointer <%p> but was <%p>"), preMessage, expected, actual);
+ CuFail(tc, buf);
+}
+
+void CuAssertPtrNotNull(CuTest* tc, const TCHAR* preMessage, const void* pointer) {
+ TCHAR buf[STRING_MAX];
+ if (pointer != NULL) return;
+ _sntprintf(buf, STRING_MAX, _T("%s : null pointer unexpected, but was <%p>"), preMessage, pointer);
+ CuFail(tc, buf);
+}
+
+void CuTestRun(CuTest* tc) {
+ // jmp_buf buf;
+ // tc->jumpBuf = &buf;
+ // if (setjmp(buf) == 0)
+ // {
+ tc->ran = 1;
+ (tc->function)(tc);
+ // }
+ // tc->jumpBuf = 0;
+}
+
+/*-------------------------------------------------------------------------*
+ * CuSuite
+ *-------------------------------------------------------------------------*/
+
+void CuSuiteInit(CuSuite* testSuite, const TCHAR *name) {
+ testSuite->name = NULL;
+ CU_TDUP(testSuite->name, name);
+ testSuite->count = 0;
+ testSuite->failCount = 0;
+ testSuite->notimplCount = 0;
+ testSuite->timeTaken = 0;
+}
+
+CuSuite* CuSuiteNew(const TCHAR *name) {
+ CuSuite* testSuite = CU_ALLOC(CuSuite);
+ CuSuiteInit(testSuite, name);
+ return testSuite;
+}
+
+void CuSuiteDelete(CuSuite* suite) {
+ free(suite->name);
+ for (int i = 0; i < suite->count; i++) {
+ CuTestDelete(suite->list[i]);
+ }
+ free(suite);
+}
+
+void CuSuiteAdd(CuSuite* testSuite, CuTest *testCase) {
+ assert(testSuite->count < MAX_TEST_CASES);
+ testSuite->list[testSuite->count] = testCase;
+ testSuite->count++;
+}
+
+void CuSuiteAddSuite(CuSuite* testSuite, CuSuite* testSuite2) {
+ int i;
+ for (i = 0; i < testSuite2->count; ++i) {
+ CuTest* testCase = testSuite2->list[i];
+ CuSuiteAdd(testSuite, testCase);
+ }
+}
+
+void CuSuiteRun(CuSuite* testSuite) {
+ int i;
+ uint64_t start = Misc::currentTimeMillis();
+ for (i = 0; i < testSuite->count; ++i) {
+ CuTest* testCase = testSuite->list[i];
+ try {
+ CuTestRun(testCase);
+ } catch (CLuceneError& err) {
+ testCase->failed = 1;
+ CuMessage(testCase, err.twhat());
+ }
+ testSuite->timeTaken = Misc::currentTimeMillis() - start;
+ if (testCase->failed) {
+ testSuite->failCount += 1;
+ }
+ if (testCase->notimpl) {
+ testSuite->notimplCount += 1;
+ }
+ }
+}
+
+void CuSuiteSummary(CuSuite* testSuite, CuString* summary, bool times) {
+ int i;
+ for (i = 0; i < testSuite->count; ++i) {
+ CuTest* testCase = testSuite->list[i];
+ CuStringAppend(summary, testCase->failed ? _T("F") :
+ testCase->notimpl ? _T("N") : _T("."));
+ }
+ if (times) {
+ int bufferLen = 25 - summary->length - 10;
+ for (int i = 0; i < bufferLen; i++)
+ CuStringAppend(summary, _T(" "));
+ CuStringAppendFormat(summary, _T(" - %dms"), testSuite->timeTaken);
+ }
+ CuStringAppend(summary, _T("\n"));
+}
+
+void CuSuiteOverView(CuSuite* testSuite, CuString* details) {
+ CuStringAppendFormat(details, _T("%d %s run: %d passed, %d failed, ")
+ _T("%d not implemented.\n"),
+ testSuite->count,
+ testSuite->count == 1 ? "test" : "tests",
+ testSuite->count - testSuite->failCount -
+ testSuite->notimplCount,
+ testSuite->failCount, testSuite->notimplCount);
+}
+
+void CuSuiteDetails(CuSuite* testSuite, CuString* details) {
+ int i;
+ int failCount = 0;
+
+ if (testSuite->failCount != 0 && verbose) {
+ CuStringAppendFormat(details, _T("\nFailed tests in %s:\n"), testSuite->name);
+ for (i = 0; i < testSuite->count; ++i) {
+ CuTest* testCase = testSuite->list[i];
+ if (testCase->failed) {
+ failCount++;
+ CuStringAppendFormat(details, _T("%d) %s: %s\n"),
+ failCount, testCase->name, testCase->message);
+ }
+ }
+ }
+ if (testSuite->notimplCount != 0 && verbose) {
+ CuStringAppendFormat(details, _T("\nNot Implemented tests in %s:\n"), testSuite->name);
+ for (i = 0; i < testSuite->count; ++i) {
+ CuTest* testCase = testSuite->list[i];
+ if (testCase->notimpl) {
+ failCount++;
+ CuStringAppendFormat(details, _T("%d) %s: %s\n"),
+ failCount, testCase->name, testCase->message);
+ }
+ }
+ }
+}
+
+/*-------------------------------------------------------------------------*
+ * CuSuiteList
+ *-------------------------------------------------------------------------*/
+
+CuSuiteList* CuSuiteListNew(const TCHAR *name) {
+ CuSuiteList* testSuite = CU_ALLOC(CuSuiteList);
+ testSuite->name = NULL;
+ CU_TDUP(testSuite->name, name);
+ testSuite->count = 0;
+ return testSuite;
+}
+
+void CuSuiteListDelete(CuSuiteList* lst) {
+ free(lst->name);
+ for (int i = 0; i < lst->count; i++) {
+ CuSuiteDelete(lst->list[i]);
+ }
+ free(lst);
+}
+
+void CuSuiteListAdd(CuSuiteList *suites, CuSuite *origsuite) {
+ assert(suites->count < MAX_TEST_CASES);
+ suites->list[suites->count] = origsuite;
+ suites->count++;
+}
+
+void CuSuiteListRun(CuSuiteList* testSuite) {
+ int i;
+ for (i = 0; i < testSuite->count; ++i) {
+ CuSuite* testCase = testSuite->list[i];
+ CuSuiteRun(testCase);
+ }
+}
+
+static const TCHAR *genspaces(int i) {
+ TCHAR *str = (TCHAR*) malloc((i + 1) * sizeof (TCHAR));
+ for (int j = 0; j < i; j++)
+ str[j] = _T(' ');
+ str[i] = '\0';
+ return str;
+}
+
+void CuSuiteListRunWithSummary(CuSuiteList* testSuite, bool verbose, bool times) {
+ int i;
+
+ _tprintf(_T("%s:\n"), testSuite->name);
+ for (i = 0; i < testSuite->count; ++i) {
+ bool hasprinted = false;
+ CuSuite* testCase = testSuite->list[i];
+ CuString *str = CuStringNew();
+
+ size_t len = _tcslen(testCase->name);
+ const TCHAR* spaces = len > 31 ? NULL : genspaces(31 - len);
+ _tprintf(_T(" %s:%s"), testCase->name, len > 31 ? _T("") : spaces);
+ free((void*) spaces);
+ fflush(stdout);
+
+ CuSuiteRun(testCase);
+ if (verbose) {
+ for (int i = 0; i < testCase->count; i++) {
+ if (testCase->list[i]->ran) {
+ if (testCase->list[i]->message != NULL) {
+ if (!hasprinted)
+ printf("\n");
+ _tprintf(_T(" %s:\n"), testCase->list[i]->name);
+
+ TCHAR* msg = testCase->list[i]->message;
+ bool nl = true;
+ //write out message, indenting on new lines
+ while (*msg != '\0') {
+ if (nl) {
+ printf(" ");
+ nl = false;
+ }
+ if (*msg == '\n')
+ nl = true;
+ putc(*msg, stdout);
+
+ msg++;
+ }
+
+ if (testCase->list[i]->message[_tcslen(testCase->list[i]->message) - 1] != '\n')
+ printf("\n");
+ hasprinted = true;
+ }
+ }
+ }
+ }
+ CuSuiteSummary(testCase, str, times);
+ if (hasprinted)
+ _tprintf(_T(" Result: %s\n"), str->buffer);
+ else
+ _tprintf(_T(" %s"), str->buffer);
+
+ CuStringFree(str);
+ }
+ _tprintf(_T("\n"));
+}
+
+int CuSuiteListDetails(CuSuiteList* testSuite, CuString* details) {
+ int i;
+ int failCount = 0;
+ int notImplCount = 0;
+ int count = 0;
+
+ for (i = 0; i < testSuite->count; ++i) {
+ failCount += testSuite->list[i]->failCount;
+ notImplCount += testSuite->list[i]->notimplCount;
+ count += testSuite->list[i]->count;
+ }
+ CuStringAppendFormat(details, _T("%d %s run: %d passed, %d failed, ")
+ _T("%d not implemented.\n"),
+ count,
+ count == 1 ? _T("test") : _T("tests"),
+ count - failCount - notImplCount,
+ failCount, notImplCount);
+
+ if (failCount != 0 && verbose) {
+ for (i = 0; i < testSuite->count; ++i) {
+ CuString *str = CuStringNew();
+ CuSuite* testCase = testSuite->list[i];
+ if (testCase->failCount) {
+ CuSuiteDetails(testCase, str);
+ CuStringAppend(details, str->buffer);
+ }
+ CuStringFree(str);
+ }
+ }
+ if (notImplCount != 0 && verbose) {
+ for (i = 0; i < testSuite->count; ++i) {
+ CuString *str = CuStringNew();
+ CuSuite* testCase = testSuite->list[i];
+ if (testCase->notimplCount) {
+ CuSuiteDetails(testCase, str);
+ CuStringAppend(details, str->buffer);
+ }
+ CuStringFree(str);
+ }
+ }
+ return failCount;
+}
+
Added: branches/lucene2_3_2/src/contribs/contribs-lib-test/CuTest.h
===================================================================
--- branches/lucene2_3_2/src/contribs/contribs-lib-test/CuTest.h (rev 0)
+++ branches/lucene2_3_2/src/contribs/contribs-lib-test/CuTest.h 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,122 @@
+/*------------------------------------------------------------------------------
+ * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
+ *
+ * Distributable under the terms of either the Apache License (Version 2.0) or
+ * the GNU Lesser General Public License, as specified in the COPYING file.
+------------------------------------------------------------------------------*/
+#ifndef CU_TEST_H
+#define CU_TEST_H
+
+/* CuString */
+
+TCHAR* CuWstrAlloc(int size);
+TCHAR* CuWstrCopy(const TCHAR* old);
+
+#define CU_ALLOC(TYPE) ((TYPE*) malloc(sizeof(TYPE)))
+#define CU_TDUP(dest,src) dest=((TCHAR*)malloc(sizeof(TCHAR)*_tcslen(src)+sizeof(TCHAR)));_tcscpy(dest,src);
+
+#define HUGE_STRING_LEN 8192
+#define STRING_MAX 256
+#define STRING_INC 256
+
+#define CLUCENE_ASSERT(x) CuAssert(tc,_T("Assert Failed: ") _T(#x),x)
+
+typedef struct {
+ int length;
+ int size;
+ TCHAR* buffer;
+} CuString;
+
+void CuStringInit(CuString* str);
+CuString* CuStringNew(void);
+void CuStringFree(CuString* str);
+void CuStringRead(CuString* str, TCHAR* path);
+void CuStringAppend(CuString* str, const TCHAR* text);
+void CuStringAppendChar(CuString* str, TCHAR ch);
+void CuStringAppendFormat(CuString* str, const TCHAR* format, ...);
+void CuStringResize(CuString* str, int newSize);
+
+/* CuTest */
+
+typedef struct CuTest CuTest;
+
+typedef void (*TestFunction)(CuTest *);
+
+struct CuTest {
+ TCHAR* name;
+ TestFunction function;
+ int notimpl;
+ int failed;
+ int ran;
+ TCHAR* message;
+ // jmp_buf *jumpBuf;
+};
+
+
+void CuInit(int argc, char *argv[]);
+void CuTestInit(CuTest* t, const TCHAR* name, TestFunction function);
+CuTest* CuTestNew(const TCHAR* name, TestFunction function);
+void CuTestDelete(CuTest* tst);
+void CuFail(CuTest* tc, const TCHAR* format, ...);
+void CuMessage(CuTest* tc, const TCHAR* message, ...);
+void CuMessageV(CuTest* tc, const TCHAR* format, va_list& argp);
+void CuMessageA(CuTest* tc, const char* format, ...);
+void CuNotImpl(CuTest* tc, const TCHAR* message);
+void CuAssert(CuTest* tc, const TCHAR* message, int condition);
+void CuAssertTrue(CuTest* tc, int condition);
+void CuAssertStrEquals(CuTest* tc, const TCHAR* preMessage, const TCHAR* expected, const TCHAR* actual);
+void CuAssertIntEquals(CuTest* tc, const TCHAR* preMessage, int expected, int actual);
+void CuAssertPtrEquals(CuTest* tc, const TCHAR* preMessage, const void* expected, const void* actual);
+void CuAssertPtrNotNull(CuTest* tc, const TCHAR* preMessage, const void* pointer);
+
+void CuTestRun(CuTest* tc);
+
+/* CuSuite */
+
+#define MAX_TEST_CASES 1024
+
+#define SUITE_ADD_TEST(SUITE,TEST) CuSuiteAdd(SUITE, CuTestNew(_T(#TEST), TEST))
+
+extern char clucene_data_location[1024];
+
+typedef struct {
+ TCHAR *name;
+ int count;
+ CuTest * list[MAX_TEST_CASES];
+ int failCount;
+ int notimplCount;
+ uint64_t timeTaken;
+} CuSuite;
+
+
+void CuSuiteInit(CuSuite* testSuite, const TCHAR* name);
+CuSuite* CuSuiteNew(const TCHAR* name);
+void CuSuiteDelete(CuSuite* suite);
+void CuSuiteAdd(CuSuite* testSuite, CuTest *testCase);
+void CuSuiteAddSuite(CuSuite* testSuite, CuSuite* testSuite2);
+void CuSuiteRun(CuSuite* testSuite);
+void CuSuiteSummary(CuSuite* testSuite, CuString* summary, bool times);
+void CuSuiteOverView(CuSuite* testSuite, CuString* details);
+void CuSuiteDetails(CuSuite* testSuite, CuString* details);
+
+typedef struct {
+ TCHAR *name;
+ int count;
+ CuSuite * list[MAX_TEST_CASES];
+} CuSuiteList;
+
+struct unittest {
+ const char *testname;
+ CuSuite * (*func)(void);
+};
+
+CuSuiteList* CuSuiteListNew(const TCHAR* name);
+void CuSuiteListDelete(CuSuiteList* lst);
+void CuSuiteListAdd(CuSuiteList* testSuite, CuSuite *testCase);
+void CuSuiteListRun(CuSuiteList* testSuite);
+void CuSuiteListRunWithSummary(CuSuiteList* testSuite, bool verbose, bool times);
+//void CuSuiteListSummary(CuSuiteList* testSuite, CuString* summary);
+/* Print details of test suite results; returns total number of
+ * tests which failed. */
+int CuSuiteListDetails(CuSuiteList* testSuite, CuString* details);
+#endif /* CU_TEST_H */
Added: branches/lucene2_3_2/src/contribs/contribs-lib-test/TestAnalysis.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs/contribs-lib-test/TestAnalysis.cpp (rev 0)
+++ branches/lucene2_3_2/src/contribs/contribs-lib-test/TestAnalysis.cpp 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,199 @@
+/**
+ * Copyright 2003-2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "test.h"
+#include "CLucene/analysis/cjk/CJKAnalyzer.h"
+#include "CLucene/analysis/LanguageBasedAnalyzer.h"
+#include "CLucene/snowball/SnowballFilter.h"
+
+#include <fcntl.h>
+#ifdef _CL_HAVE_IO_H
+#include <io.h>
+#endif
+#ifdef _CL_HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef _CL_HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef _CL_HAVE_DIRECT_H
+#include <direct.h>
+#endif
+#include <errno.h>
+
+CL_NS_USE2(analysis, cjk)
+CL_NS_USE2(analysis, snowball)
+
+void test(CuTest *tc, char* orig, Reader* reader, bool verbose, int64_t bytes) {
+ StandardAnalyzer analyzer;
+ TokenStream* stream = analyzer.tokenStream(NULL, reader);
+
+ uint64_t start = Misc::currentTimeMillis();
+
+ int32_t count = 0;
+ Token t;
+ char atmp[LUCENE_MAX_WORD_LEN + 1];
+ TCHAR ttmp[LUCENE_MAX_WORD_LEN + 1];
+ for (; stream->next(&t);) {
+ if (verbose) {
+ CuMessage(tc, _T("Text=%s start=%d end=%d\n"), t.termBuffer(), t.startOffset(), t.endOffset());
+ }
+ int len = t.termLength();
+
+ //use the lucene strlwr function (so copy to TCHAR first then back)
+ strncpy(atmp, orig + t.startOffset(), len);
+ atmp[len] = 0;
+ STRCPY_AtoT(ttmp, atmp, len + 1);
+ _tcslwr(ttmp);
+
+ if (_tcsncmp(t.termBuffer(), ttmp, len) != 0) {
+ TCHAR err[1024];
+ _sntprintf(err, 1024, _T("token '%s' didnt match original text at %d-%d"), t.termBuffer(), t.startOffset(), t.endOffset());
+ CuAssert(tc, err, false);
+ }
+
+ // _CLDELETE(t);
+ count++;
+ }
+
+ uint64_t end = Misc::currentTimeMillis();
+ int64_t time = end - start;
+ CuMessageA(tc, "%d milliseconds to extract ", time);
+ CuMessageA(tc, "%d tokens\n", count);
+ CuMessageA(tc, "%f microseconds/token\n", (time * 1000.0) / count);
+ CuMessageA(tc, "%f megabytes/hour\n", (bytes * 1000.0 * 60.0 * 60.0) / (time * 1000000.0));
+
+ _CLDELETE(stream);
+}
+
+void _testFile(CuTest *tc, const char* fname, bool verbose) {
+ struct fileStat buf;
+ fileStat(fname, &buf);
+ int64_t bytes = buf.st_size;
+
+ char* orig = _CL_NEWARRAY(char, bytes);
+ {
+ FILE* f = fopen(fname, "rb");
+ int64_t r = fread(orig, bytes, 1, f);
+ fclose(f);
+ }
+
+ CuMessageA(tc, " Reading test file containing %d bytes.\n", bytes);
+ jstreams::FileReader fr(fname, "ASCII");
+ const TCHAR *start;
+ size_t total = 0;
+ int32_t numRead;
+ do {
+ numRead = fr.read(start, 1, 0);
+ if (numRead == -1)
+ break;
+ total += numRead;
+ } while (numRead >= 0);
+
+ jstreams::FileReader reader(fname, "ASCII");
+
+ test(tc, orig, &reader, verbose, total);
+
+ _CLDELETE_CaARRAY(orig);
+}
+
+void testFile(CuTest *tc) {
+ char loc[1024];
+ strcpy(loc, clucene_data_location);
+ strcat(loc, "/reuters-21578/feldman-cia-worldfactbook-data.txt");
+ CuAssert(tc, _T("reuters-21578/feldman-cia-worldfactbook-data.txt does not exist"), Misc::dir_Exists(loc));
+
+ _testFile(tc, loc, false);
+}
+
+void _testCJK(CuTest *tc, const char* astr, const char** results, bool ignoreSurrogates = true) {
+ SimpleInputStreamReader r(new AStringReader(astr), SimpleInputStreamReader::UTF8);
+
+ CJKTokenizer* tokenizer = _CLNEW CJKTokenizer(&r);
+ tokenizer->setIgnoreSurrogates(ignoreSurrogates);
+ int pos = 0;
+ Token tok;
+ TCHAR tres[LUCENE_MAX_WORD_LEN];
+
+ while (results[pos] != NULL) {
+ CLUCENE_ASSERT(tokenizer->next(&tok) == true);
+
+ lucene_utf8towcs(tres, results[pos], LUCENE_MAX_WORD_LEN);
+ CuAssertStrEquals(tc, _T("unexpected token value"), tres, tok.termBuffer());
+
+ pos++;
+ }
+ CLUCENE_ASSERT(!tokenizer->next(&tok));
+
+ _CLDELETE(tokenizer);
+}
+
+void testCJK(CuTest *tc) {
+ //utf16 test
+ //we have a very large unicode character:
+ //xEFFFF = utf8(F3 AF BF BF) = utf16(DB7F DFFF) = utf8(ED AD BF, ED BF BF)
+ static const char* exp3[4] = {"\xED\xAD\xBF\xED\xBF\xBF\xe5\x95\xa4", "\xe5\x95\xa4\xED\xAD\xBF\xED\xBF\xBF", "", NULL};
+ _testCJK(tc, "\xED\xAD\xBF\xED\xBF\xBF\xe5\x95\xa4\xED\xAD\xBF\xED\xBF\xBF", exp3, false);
+
+ static const char* exp1[5] = {"test", "t\xc3\xbcrm", "values", NULL};
+ _testCJK(tc, "test t\xc3\xbcrm values", exp1);
+
+ static const char* exp2[6] = {"a", "\xe5\x95\xa4\xe9\x85\x92", "\xe9\x85\x92\xe5\x95\xa4", "", "x", NULL};
+ _testCJK(tc, "a\xe5\x95\xa4\xe9\x85\x92\xe5\x95\xa4x", exp2);
+}
+
+void testLanguageBasedAnalyzer(CuTest* tc) {
+ LanguageBasedAnalyzer a;
+ CL_NS(util)::StringReader reader(_T("he abhorred accentueren"));
+ reader.mark(50);
+ TokenStream* ts;
+ Token t;
+
+ //test with english
+ a.setLanguage(_T("English"));
+ a.setStem(false);
+ ts = a.tokenStream(_T("contents"), &reader);
+
+ CLUCENE_ASSERT(ts->next(&t));
+ CLUCENE_ASSERT(_tcscmp(t.termBuffer(), _T("he")) == 0);
+ CLUCENE_ASSERT(ts->next(&t));
+ CLUCENE_ASSERT(_tcscmp(t.termBuffer(), _T("abhorred")) == 0);
+ _CLDELETE(ts);
+
+ //now test with dutch
+ reader.reset(0);
+ a.setLanguage(_T("Dutch"));
+ a.setStem(true);
+ ts = a.tokenStream(_T("contents"), &reader);
+
+ CLUCENE_ASSERT(ts->next(&t));
+ CLUCENE_ASSERT(_tcscmp(t.termBuffer(), _T("he")) == 0);
+ CLUCENE_ASSERT(ts->next(&t));
+ CLUCENE_ASSERT(_tcscmp(t.termBuffer(), _T("abhorred")) == 0);
+ CLUCENE_ASSERT(ts->next(&t));
+ CLUCENE_ASSERT(_tcscmp(t.termBuffer(), _T("accentuer")) == 0);
+ _CLDELETE(ts);
+}
+
+CuSuite *testanalysis(void) {
+ CuSuite *suite = CuSuiteNew(_T("CLucene Analysis Test"));
+
+ SUITE_ADD_TEST(suite, testFile);
+ SUITE_ADD_TEST(suite, testCJK);
+ SUITE_ADD_TEST(suite, testLanguageBasedAnalyzer);
+
+ return suite;
+}
+// EOF
Added: branches/lucene2_3_2/src/contribs/contribs-lib-test/TestHighlight.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs/contribs-lib-test/TestHighlight.cpp (rev 0)
+++ branches/lucene2_3_2/src/contribs/contribs-lib-test/TestHighlight.cpp 2009-03-30 21:38:58 UTC (rev 2977)
@@ -0,0 +1,263 @@
+#include "test.h"
+
+#include "CLucene/store/RAMDirectory.h"
+#include "CLucene/highlighter/QueryTermExtractor.h"
+#include "CLucene/highlighter/QueryScorer.h"
+#include "CLucene/highlighter/Highlighter.h"
+#include "CLucene/highlighter/TokenGroup.h"
+#include "CLucene/highlighter/SimpleHTMLFormatter.h"
+#include "CLucene/highlighter/SimpleFragmenter.h"
+
+CL_NS_USE2(search, highlight);
+
+RAMDirectory hl_ramDir;
+StandardAnalyzer hl_analyzer;
+
+
+const TCHAR* hl_FIELD_NAME = _T("contents");
+Query* hl_originalquery = NULL;
+Query* hl_query = NULL;
+Query* hl_rewrittenquery = NULL;
+IndexReader* hl_reader = NULL;
+Searcher* hl_searcher = NULL;
+Hits* hl_hits = NULL;
+
+class hl_formatterCls : public Formatter {
+public:
+ int numHighlights;
+
+ hl_formatterCls() {
+ numHighlights = 0;
+ }
+
+ ~hl_formatterCls() {
+ }
+
+ TCHAR* highlightTerm(const TCHAR* originalText, const TokenGroup* group) {
+ if (group->getTotalScore() <= 0) {
+ return STRDUP_TtoT(originalText);
+ }
+ numHighlights++; //update stats used in assertions
+
+ int len = _tcslen(originalText) + 7;
+ TCHAR* ret = _CL_NEWARRAY(TCHAR, len + 1);
+ _tcscpy(ret, _T("<b>"));
+ _tcscat(ret, originalText);
+ _tcscat(ret, _T("</b>"));
+
+ return ret;
+ }
+};
+hl_formatterCls hl_formatter;
+
+
+const TCHAR* hl_texts[6] ={
+ _T("Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot"),
+ _T("This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy"),
+ _T("JFK has been shot"),
+ _T("John Kennedy has been shot"),
+ _T("This text has a typo in referring to Keneddy"),
+ NULL
+};
+
+void doStandardHighlights(CuTest* tc) {
+ QueryScorer scorer(hl_query);
+ Highlighter highlighter(&hl_formatter, &scorer);
+ SimpleFragmenter frag(20);
+ highlighter.setTextFragmenter(&frag);
+
+ for (int i = 0; i < hl_hits->length(); i++) {
+ const TCHAR* text = hl_hits->doc(i).get(hl_FIELD_NAME);
+ int maxNumFragmentsRequired = 2;
+ const TCHAR* fragmentSeparator = _T("...");
+ StringReader reader(text);
+ TokenStream* tokenStream = hl_analyzer.tokenStream(hl_FIELD_NAME, &reader);
+
+ TCHAR* result =
+ highlighter.getBestFragments(
+ tokenStream,
+ text,
+ maxNumFragmentsRequired,
+ fragmentSeparator);
+
+ CuMessage(tc, _T("%s\n"), result == NULL ? _T("") : result);
+ _CLDELETE_CARRAY(result);
+ _CLDELETE(tokenStream);
+ }
+}
+
+void doSearching(CuTest* tc, const TCHAR* queryString) {
+ if (hl_searcher == NULL)
+ hl_searcher = _CLNEW IndexSearcher(&hl_ramDir);
+
+ if (hl_rewrittenquery != NULL && hl_originalquery != NULL) {
+ if (hl_originalquery != hl_rewrittenquery)
+ _CLDELETE(hl_rewrittenquery);
+ _CLDELETE(hl_originalquery);
+ }
+ hl_originalquery = QueryParser::parse(queryString, hl_FIELD_NAME, &hl_analyzer);
+
+ //for any multi-term queries to work (prefix, wildcard, range,fuzzy etc) you must use a rewritten query!
+ hl_rewrittenquery = hl_originalquery->rewrite(hl_reader);
+ hl_query = hl_rewrittenquery;
+
+ TCHAR* s = hl_originalquery->toString(hl_FIELD_NAME);
+ CuMessage(tc, _T("Searching for: %s\n"), s == NULL ? _T("") : s);
+ _CLDELETE_CARRAY(s);
+
+ s = hl_rewrittenquery->toString(hl_FIELD_NAME);
+ CuMessage(tc, _T("Rewritten query: %s\n"), s == NULL ? _T("") : s);
+ _CLDELETE_CARRAY(s);
+
+ if (hl_hits != NULL)
+ _CLDELETE(hl_hits);
+ hl_hits = hl_searcher->search(hl_query);
+ hl_formatter.numHighlights = 0;
+}
+
+void testSimpleHighlighter(CuTest *tc) {
+ doSearching(tc, _T("Kennedy"));
+ QueryScorer scorer(hl_query);
+ Highlighter highlighter(&scorer);
+ SimpleFragmenter fragmenter(40);
+
+ highlighter.setTextFragmenter(&fragmenter);
+ int maxNumFragmentsRequired = 2;
+ for (int i = 0; i < hl_hits->length(); i++) {
+ const TCHAR* text = hl_hits->doc(i).get(hl_FIELD_NAME);
+ StringReader reader(text);
+ TokenStream* tokenStream = hl_analyzer.tokenStream(hl_FIELD_NAME, &reader);
+
+ TCHAR* result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, _T("..."));
+ CuMessage(tc, _T("%s\n"), result == NULL ? _T("") : result);
+ _CLDELETE_CARRAY(result);
+ _CLDELETE(tokenStream);
+ }
+ //Not sure we can assert anything here - just running to check we dont throw any exceptions
+}
+
+void testGetFuzzyFragments(CuTest *tc) {
+ doSearching(tc, _T("Kinnedy~"));
+ doStandardHighlights(tc);
+
+ TCHAR msg[1024];
+ _sntprintf(msg, 1024, _T("Failed to find correct number of highlights %d found"), hl_formatter.numHighlights);
+ CuAssert(tc, msg, hl_formatter.numHighlights == 5);
+}
+
+void testGetWildCardFragments(CuTest *tc) {
+ doSearching(tc, _T("K?nnedy"));
+ doStandardHighlights(tc);
+
+ TCHAR msg[1024];
+ _sntprintf(msg, 1024, _T("Failed to find correct number of highlights %d found"), hl_formatter.numHighlights);
+ CuAssert(tc, msg, hl_formatter.numHighlights == 4);
+}
+
+void testGetBestFragmentsSimpleQuery(CuTest *tc) {
+ doSearching(tc, _T("Kennedy"));
+ doStandardHighlights(tc);
+
+ TCHAR msg[1024];
+ _sntprintf(msg, 1024, _T("Failed to find correct number of highlights %d found"), hl_formatter.numHighlights);
+ CuAssert(tc, msg, hl_formatter.numHighlights == 4);
+}
+
+void testGetMidWildCardFragments(CuTest *tc) {
+ doSearching(tc, _T("K*dy"));
+ doStandardHighlights(tc);
+
+ TCHAR msg[1024];
+ _sntprintf(msg, 1024, _T("Failed to find correct number of highlights %d found"), hl_formatter.numHighlights);
+ CuAssert(tc, msg, hl_formatter.numHighlights == 5);
+}
+
+void testGetBestFragmentsPhrase(CuTest *tc) {
+ doSearching(tc, _T("\"John Kennedy\""));
+ doStandardHighlights(tc);
+
+ TCHAR msg[1024];
+ _sntprintf(msg, 1024, _T("Failed to find correct number of highlights %d found"), hl_formatter.numHighlights);
+ CuAssert(tc, msg, hl_formatter.numHighlights == 2);
+}
+
+void testGetBestFragmentsMultiTerm(CuTest *tc) {
+ doSearching(tc, _T("John Ken*"));
+ doStandardHighlights(tc);
+
+ TCHAR msg[1024];
+ _sntprintf(msg, 1024, _T("Failed to find correct number of highlights %d found"), hl_formatter.numHighlights);
+ CuAssert(tc, msg, hl_formatter.numHighlights == 6);
+}
+
+void testGetBestFragmentsWithOr(CuTest *tc) {
+ doSearching(tc, _T("JFK OR Kennedy"));
+ doStandardHighlights(tc);
+
+ TCHAR msg[1024];
+ _sntprintf(msg, 1024, _T("Failed to find correct number of highlights %d found"), hl_formatter.numHighlights);
+ CuAssert(tc, msg, hl_formatter.numHighlights == 5);
+}
+
+void testGetRangeFragments(CuTest *tc) {
+ TCHAR qry[200];
+ _sntprintf(qry, 200, _T("%s:[Kannedy TO Kznnedy]"), hl_FIELD_NAME); //bug?needs lower case
+
+ doSearching(tc, qry);
+ doStandardHighlights(tc);
+
+ TCHAR msg[1024];
+ _sntprintf(msg, 1024, _T("Failed to find correct number of highlights %d found"), hl_formatter.numHighlights);
+ CuAssert(tc, msg, hl_formatter.numHighlights == 5);
+}
+
+void setupHighlighter(CuTest *tc) {
+ IndexWriter writer(&hl_ramDir, &hl_analyzer, true);
+ for (int i = 0; hl_texts[i] != NULL; i++) {
+ Document d;
+ d.add(*_CLNEW Field(hl_FIELD_NAME, hl_texts[i], Field::STORE_YES | Field::INDEX_TOKENIZED));
+ writer.addD...
[truncated message content] |
|
From: <ust...@us...> - 2009-03-30 21:34:24
|
Revision: 2976
http://clucene.svn.sourceforge.net/clucene/?rev=2976&view=rev
Author: ustramooner
Date: 2009-03-30 21:34:13 +0000 (Mon, 30 Mar 2009)
Log Message:
-----------
various cleanups to the streams
Modified Paths:
--------------
branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipcompressstream.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipcompressstream.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipinputstream.h
branches/lucene2_3_2/src/contribs-lib/CMakeLists.txt
Added Paths:
-----------
branches/lucene2_3_2/src/contribs-lib/CLucene/util/
branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipinputstream.cpp
Removed Paths:
-------------
branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/
branches/lucene2_3_2/src/contribs-lib/CLucene/util/Reader.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/bufferedstream.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/bz2inputstream.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/util/bz2inputstream.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/dostime.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/util/dostime.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/fileinputstream.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/util/fileinputstream.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/filereader.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/util/filereader.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipinputstream.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/util/inputstream.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/inputstreambuffer.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/inputstreamreader.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/util/inputstreamreader.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/jstreamsconfig.h.cmake
branches/lucene2_3_2/src/contribs-lib/CLucene/util/streambase.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/streamreader.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/stringreader.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/subinputstream.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/util/subinputstream.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/substreamprovider.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/tarinputstream.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/util/tarinputstream.h
branches/lucene2_3_2/src/contribs-lib/CLucene/util/zipinputstream.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/util/zipinputstream.h
Property changes on: branches/lucene2_3_2/src/contribs-lib/CLucene/util
___________________________________________________________________
Added: svn:mergeinfo
+
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/Reader.h
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/Reader.h 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/Reader.h 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,18 +0,0 @@
-/*------------------------------------------------------------------------------
-* Copyright (C) 2003-2006 Jos van den Oever
-*
-* Distributable under the terms of either the Apache License (Version 2.0) or
-* the GNU Lesser General Public License, as specified in the COPYING file.
-------------------------------------------------------------------------------*/
-#ifndef READER_H
-#define READER_H
-
-#include "streambase.h"
-
-namespace jstreams {
-
-typedef StreamBase<wchar_t> Reader;
-
-} // end namespace jstreams
-
-#endif
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/bufferedstream.h
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/bufferedstream.h 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/bufferedstream.h 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,137 +0,0 @@
-/*------------------------------------------------------------------------------
-* Copyright (C) 2003-2006 Jos van den Oever
-*
-* Distributable under the terms of either the Apache License (Version 2.0) or
-* the GNU Lesser General Public License, as specified in the COPYING file.
-------------------------------------------------------------------------------*/
-#ifndef BUFFEREDSTREAM_H
-#define BUFFEREDSTREAM_H
-
-#include "streambase.h"
-#include "inputstreambuffer.h"
-#include <cassert>
-
-namespace jstreams {
-
-template <class T>
-class BufferedInputStream : public StreamBase<T> {
-private:
- bool finishedWritingToBuffer;
- InputStreamBuffer<T> buffer;
-
- void writeToBuffer(int32_t minsize);
- int32_t read_(const T*& start, int32_t min, int32_t max);
-protected:
- /**
- * This function must be implemented by the subclasses.
- * It should write a maximum of @p space characters at the buffer
- * position pointed to by @p start. If no more data is avaiable due to
- * end of file, -1 should be returned. If an error occurs, the status
- * should be set to Error, an error message should be set and the function
- * must return -1.
- **/
- virtual int32_t fillBuffer(T* start, int32_t space) = 0;
- // this function might be useful if you want to reuse a bufferedstream
- void resetBuffer() {printf("implement 'resetBuffer'\n");}
- BufferedInputStream<T>();
-public:
- int32_t read(const T*& start, int32_t min, int32_t max);
- int64_t reset(int64_t);
- virtual int64_t skip(int64_t ntoskip);
-};
-
-template <class T>
-BufferedInputStream<T>::BufferedInputStream() {
- finishedWritingToBuffer = false;
-}
-
-template <class T>
-void
-BufferedInputStream<T>::writeToBuffer(int32_t ntoread) {
- int32_t missing = ntoread - buffer.avail;
- int32_t nwritten = 0;
- while (missing > 0 && nwritten >= 0) {
- int32_t space;
- space = buffer.makeSpace(missing);
- T* start = buffer.readPos + buffer.avail;
- nwritten = fillBuffer(start, space);
- assert(StreamBase<T>::status != Eof);
- if (nwritten > 0) {
- buffer.avail += nwritten;
- missing = ntoread - buffer.avail;
- }
- }
- if (nwritten < 0) {
- finishedWritingToBuffer = true;
- }
-}
-template <class T>
-int32_t
-BufferedInputStream<T>::read(const T*& start, int32_t min, int32_t max) {
- if (StreamBase<T>::status == Error) return -2;
- if (StreamBase<T>::status == Eof) return -1;
-
- // do we need to read data into the buffer?
- if (!finishedWritingToBuffer && min > buffer.avail) {
- // do we have enough space in the buffer?
- writeToBuffer(min);
- if (StreamBase<T>::status == Error) return -2;
- }
-
- int32_t nread = buffer.read(start, max);
-
- BufferedInputStream<T>::position += nread;
- if (BufferedInputStream<T>::position > BufferedInputStream<T>::size
- && BufferedInputStream<T>::size > 0) {
- // error: we read more than was specified in size
- // this is an error because all dependent code might have been labouring
- // under a misapprehension
- BufferedInputStream<T>::status = Error;
- BufferedInputStream<T>::error = "Stream is longer than specified.";
- nread = -2;
- } else if (BufferedInputStream<T>::status == Ok && buffer.avail == 0
- && finishedWritingToBuffer) {
- BufferedInputStream<T>::status = Eof;
- if (BufferedInputStream<T>::size == -1) {
- BufferedInputStream<T>::size = BufferedInputStream<T>::position;
- }
- // save one call to read() by already returning -1 if no data is there
- if (nread == 0) nread = -1;
- }
- return nread;
-}
-template <class T>
-int64_t
-BufferedInputStream<T>::reset(int64_t newpos) {
- assert(newpos >= 0);
- if (StreamBase<T>::status == Error) return -2;
- // check to see if we have this position
- int64_t d = BufferedInputStream<T>::position - newpos;
- if (buffer.readPos - d >= buffer.start && -d < buffer.avail) {
- BufferedInputStream<T>::position -= d;
- buffer.avail += d;
- buffer.readPos -= d;
- StreamBase<T>::status = Ok;
- }
- return StreamBase<T>::position;
-}
-template <class T>
-int64_t
-BufferedInputStream<T>::skip(int64_t ntoskip) {
- const T *begin;
- int32_t nread;
- int64_t skipped = 0;
- while (ntoskip) {
- int32_t step = (int32_t)((ntoskip > buffer.size) ?buffer.size :ntoskip);
- nread = read(begin, 1, step);
- if (nread <= 0) {
- return skipped;
- }
- ntoskip -= nread;
- skipped += nread;
- }
- return skipped;
-}
-}
-
-#endif
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/bz2inputstream.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/bz2inputstream.cpp 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/bz2inputstream.cpp 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,138 +0,0 @@
-/*------------------------------------------------------------------------------
-* Copyright (C) 2003-2006 Jos van den Oever
-*
-* Distributable under the terms of either the Apache License (Version 2.0) or
-* the GNU Lesser General Public License, as specified in the COPYING file.
-------------------------------------------------------------------------------*/
-#include "CLucene/jstreams/jstreamsconfig.h"
-#include "bz2inputstream.h"
-using namespace jstreams;
-
-bool
-BZ2InputStream::checkHeader(const char* data, int32_t datasize) {
- static const char magic[] = {0x42, 0x5a, 0x68, 0x39, 0x31};
- if (datasize < 5) return false;
- return memcmp(data, magic, 5) == 0;
-}
-BZ2InputStream::BZ2InputStream(StreamBase<char>* input) {
- // initialize values that signal state
- this->input = input;
-
- // TODO: check first bytes of stream before allocating buffer
- // 0x42 0x5a 0x68 0x39 0x31
- if (!checkMagic()) {
- error = "Magic bytes are wrong.";
- status = Error;
- allocatedBz = false;
- return;
- }
-
- bzstream = (bz_stream*)malloc(sizeof(bz_stream));
- bzstream->bzalloc = NULL;
- bzstream->bzfree = NULL;
- bzstream->opaque = NULL;
- bzstream->avail_in = 0;
- bzstream->next_in = NULL;
- int r;
- r = BZ2_bzDecompressInit(bzstream, 1, 0);
- if (r != BZ_OK) {
- error = "Error initializing BZ2InputStream.";
- fprintf(stderr, "Error initializing BZ2InputStream.\n");
- dealloc();
- status = Error;
- return;
- }
- allocatedBz = true;
- // signal that we need to read into the buffer
- bzstream->avail_out = 1;
-
- // set the minimum size for the output buffer
- mark(262144);
-}
-BZ2InputStream::~BZ2InputStream() {
- dealloc();
-}
-void
-BZ2InputStream::dealloc() {
- if (allocatedBz) {
- BZ2_bzDecompressEnd(bzstream);
- free(bzstream);
- bzstream = 0;
- }
-}
-bool
-BZ2InputStream::checkMagic() {
- const char* begin;
- int32_t nread;
-
- int64_t pos = input->getPosition();
- nread = input->read(begin, 5, 5);
- input->reset(pos);
- if (nread != 5) {
- return false;
- }
-
- return checkHeader(begin, 5);
-}
-void
-BZ2InputStream::readFromStream() {
- // read data from the input stream
- const char* inStart;
- int32_t nread;
- nread = input->read(inStart, 1, 0);
- if (nread <= -1) {
- status = Error;
- error = input->getError();
- } else if (nread < 1) {
- status = Error;
- error = "unexpected end of stream";
- } else {
- bzstream->next_in = (char*)inStart;
- bzstream->avail_in = nread;
- }
-}
-int32_t
-BZ2InputStream::fillBuffer(char* start, int32_t space) {
- if (bzstream == 0) return -1;
- // make sure there is data to decompress
- if (bzstream->avail_out != 0) {
- readFromStream();
- if (status != Ok) {
- // no data was read
- return -1;
- }
- }
- // make sure we can write into the buffer
- bzstream->avail_out = space;
- bzstream->next_out = start;
- // decompress
- int r = BZ2_bzDecompress(bzstream);
- // inform the buffer of the number of bytes that was read
- int32_t nwritten = space - bzstream->avail_out;
- switch (r) {
- case BZ_PARAM_ERROR:
- error = "BZ_PARAM_ERROR";
- status = Error;
- return -1;
- case BZ_DATA_ERROR:
- error = "BZ_DATA_ERROR";
- status = Error;
- return -1;
- case BZ_DATA_ERROR_MAGIC:
- error = "BZ_DATA_ERROR_MAGIC";
- status = Error;
- return -1;
- case BZ_MEM_ERROR:
- error = "BZ_MEM_ERROR";
- status = Error;
- return -1;
- case BZ_STREAM_END:
- if (bzstream->avail_in) {
- input->reset(input->getPosition()-bzstream->avail_in);
- }
- // we are finished decompressing,
- // (but this stream is not yet finished)
- dealloc();
- }
- return nwritten;
-}
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/bz2inputstream.h
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/bz2inputstream.h 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/bz2inputstream.h 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,35 +0,0 @@
-/*------------------------------------------------------------------------------
-* Copyright (C) 2003-2006 Jos van den Oever
-*
-* Distributable under the terms of either the Apache License (Version 2.0) or
-* the GNU Lesser General Public License, as specified in the COPYING file.
-------------------------------------------------------------------------------*/
-#ifndef BZ2INPUTSTREAM_H
-#define BZ2INPUTSTREAM_H
-
-#include "bufferedstream.h"
-
-#include <bzlib.h>
-
-namespace jstreams {
-
-class BZ2InputStream : public BufferedInputStream<char> {
-private:
- bool allocatedBz;
- bz_stream* bzstream;
- StreamBase<char> *input;
-
- void dealloc();
- void readFromStream();
- bool checkMagic();
-protected:
- int32_t fillBuffer(char* start, int32_t space);
-public:
- BZ2InputStream(StreamBase<char>* input);
- ~BZ2InputStream();
- static bool checkHeader(const char* data, int32_t datasize);
-};
-
-} // end namespace jstreams
-
-#endif
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/dostime.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/dostime.cpp 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/dostime.cpp 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,71 +0,0 @@
-/*------------------------------------------------------------------------------
-* Copyright (C) 2003-2006 Jos van den Oever
-*
-* Distributable under the terms of either the Apache License (Version 2.0) or
-* the GNU Lesser General Public License, as specified in the COPYING file.
-------------------------------------------------------------------------------*/
-/* dostime.c - convert dos time to/from time_t.
-
- Copyright (C) 2002 Free Software Foundation
-
- This program is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License
- as published by the Free Software Foundation; either version 2
- of the License, or (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-*/
-
-// #include <config.h>
-
-#include <time.h>
-
-#include <stddef.h>
-
-#include "dostime.h"
-
-/*
- * The specification to which this was written. From Joe Buck.
- * The DOS format appears to have only 2 second resolution. It is an
- * unsigned long, and ORs together
- *
- * (year-1980)<<25
- * month<<21 (month is tm_mon + 1, 1=Jan through 12=Dec)
- * day<<16 (day is tm_mday, 1-31)
- * hour<<11 (hour is tm_hour, 0-23)
- * min<<5 (min is tm_min, 0-59)
- * sec>>1 (sec is tm_sec, 0-59, that's right, we throw away the LSB)
- *
- * DOS uses local time, so the localtime() call is used to turn the time_t
- * into a struct tm.
- */
-
-time_t
-dos2unixtime (unsigned long dostime)
-{
- struct tm ltime;
- time_t now = time (NULL);
-
- /* Call localtime to initialize timezone in TIME. */
- ltime = *localtime (&now);
-
- ltime.tm_year = (dostime >> 25) + 80;
- ltime.tm_mon = ((dostime >> 21) & 0x0f) - 1;
- ltime.tm_mday = (dostime >> 16) & 0x1f;
- ltime.tm_hour = (dostime >> 11) & 0x0f;
- ltime.tm_min = (dostime >> 5) & 0x3f;
- ltime.tm_sec = (dostime & 0x1f) << 1;
-
- ltime.tm_wday = -1;
- ltime.tm_yday = -1;
- ltime.tm_isdst = -1;
-
- return mktime (<ime);
-}
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/dostime.h
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/dostime.h 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/dostime.h 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,27 +0,0 @@
-/*------------------------------------------------------------------------------
-* Copyright (C) 2003-2006 Jos van den Oever
-*
-* Distributable under the terms of either the Apache License (Version 2.0) or
-* the GNU Lesser General Public License, as specified in the COPYING file.
-------------------------------------------------------------------------------*/
-/*
- dostime.h - function prototypes
- Copyright (C) 1999, 2002 Bryan Burns
-
- This program is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License
- as published by the Free Software Foundation; either version 2
- of the License, or (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-
-time_t dos2unixtime(unsigned long dostime);
-unsigned long unix2dostime(time_t*);
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/fileinputstream.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/fileinputstream.cpp 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/fileinputstream.cpp 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,78 +0,0 @@
-/*------------------------------------------------------------------------------
-* Copyright (C) 2003-2006 Jos van den Oever
-*
-* Distributable under the terms of either the Apache License (Version 2.0) or
-* the GNU Lesser General Public License, as specified in the COPYING file.
-------------------------------------------------------------------------------*/
-#include "CLucene/jstreams/jstreamsconfig.h"
-#include "fileinputstream.h"
-#include <cerrno>
-#include <cstring>
-using namespace jstreams;
-
-const int32_t FileInputStream::defaultBufferSize = 1048576;
-FileInputStream::FileInputStream(const char *filepath, int32_t buffersize) {
- // try to open the file for reading
- file = fopen(filepath, "rb");
- this->filepath = filepath;
- if (file == 0) {
- // handle error
- error = "Could not read file '";
- error += filepath;
- error += "': ";
- error += strerror(errno);
- status = Error;
- return;
- }
- // determine file size. if the stream is not seekable, the size will be -1
- fseek(file, 0, SEEK_END);
- size = ftell(file);
- fseek(file, 0, SEEK_SET);
-
- // if the file has size 0, make sure that it's really empty
- // this is useful for filesystems like /proc that report files as size 0
- // for files that do contain content
- if (size == 0) {
- char dummy[1];
- size_t n = fread(dummy, 1, 1, file);
- if (n == 1) {
- size = -1;
- fseek(file, 0, SEEK_SET);
- } else {
- fclose(file);
- file = 0;
- return;
- }
- }
-
- // allocate memory in the buffer
- int32_t bufsize = (size <= buffersize) ?size+1 :buffersize;
- mark(bufsize);
-}
-FileInputStream::~FileInputStream() {
- if (file) {
- if (fclose(file)) {
- // handle error
- error = "Could not close file '" + filepath + "'.";
- }
- }
-}
-int32_t
-FileInputStream::fillBuffer(char* start, int32_t space) {
- if (file == 0) return -1;
- // read into the buffer
- int32_t nwritten = fread(start, 1, space, file);
- // check the file stream status
- if (ferror(file)) {
- error = "Could not read from file '" + filepath + "'.";
- fclose(file);
- file = 0;
- status = Error;
- return -1;
- }
- if (feof(file)) {
- fclose(file);
- file = 0;
- }
- return nwritten;
-}
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/fileinputstream.h
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/fileinputstream.h 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/fileinputstream.h 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,30 +0,0 @@
-/*------------------------------------------------------------------------------
-* Copyright (C) 2003-2006 Jos van den Oever
-*
-* Distributable under the terms of either the Apache License (Version 2.0) or
-* the GNU Lesser General Public License, as specified in the COPYING file.
-------------------------------------------------------------------------------*/
-#ifndef FILEINPUTSTREAM_H
-#define FILEINPUTSTREAM_H
-
-#include "bufferedstream.h"
-
-namespace jstreams {
-
-class FileInputStream : public BufferedInputStream<char> {
-private:
- FILE *file;
- std::string filepath;
-
-public:
- static const int32_t defaultBufferSize;
- explicit FileInputStream(const char *filepath,
- int32_t buffersize=defaultBufferSize);
- ~FileInputStream();
- int32_t fillBuffer(char* start, int32_t space);
-};
-
-} // end namespace jstreams
-
-#endif
-
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/filereader.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/filereader.cpp 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/filereader.cpp 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,37 +0,0 @@
-#include "CLucene/jstreams/jstreamsconfig.h"
-#include "filereader.h"
-#include "fileinputstream.h"
-#include "inputstreamreader.h"
-using namespace jstreams;
-
-FileReader::FileReader(const char* fname, const char* encoding_scheme,
- int32_t cachelen, int32_t /*cachebuff*/) {
- input = new FileInputStream(fname, cachelen);
- reader = new InputStreamReader(input, encoding_scheme);
-}
-FileReader::~FileReader() {
- if (reader) delete reader;
- if (input) delete input;
-}
-int32_t
-FileReader::read(const wchar_t*& start, int32_t min, int32_t max) {
- int32_t nread = reader->read(start, min, max);
- if (nread < -1) {
- error = reader->getError();
- status = Error;
- return nread;
- } else if (nread == -1) {
- status = Eof;
- }
- return nread;
-}
-int64_t
-FileReader::reset(int64_t newpos) {
- position = reader->reset(newpos);
- if (position < -1) {
- status = Error;
- error = reader->getError();
- }
- return position;
-}
-
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/filereader.h
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/filereader.h 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/filereader.h 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,24 +0,0 @@
-#ifndef FILEREADER_H
-#define FILEREADER_H
-
-#include "streambase.h"
-
-namespace jstreams {
-
-class FileInputStream;
-class InputStreamReader;
-class FileReader : public StreamBase<wchar_t> {
- FileInputStream* input;
- InputStreamReader* reader;
-public:
- explicit FileReader(const char* fname, const char* encoding_scheme=NULL,
- const int32_t cachelen = 13,
- const int32_t cachebuff = 14 );
- ~FileReader();
- int32_t read(const wchar_t*& start, int32_t min, int32_t max);
- int64_t reset(int64_t);
-};
-
-} // end namespace jstreams
-
-#endif
Modified: branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipcompressstream.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/gzipcompressstream.cpp 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipcompressstream.cpp 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,108 +1,120 @@
-#include "CLucene/jstreams/jstreamsconfig.h"
+#include "CLucene/_ApiHeader.h"
#include "gzipcompressstream.h"
+#include "CLucene/util/_bufferedstream.h"
#include <zlib.h>
+CL_NS_DEF(util)
-using namespace jstreams;
+class GZipCompressInputStream::Internal: public CL_NS(util)::BufferedInputStreamImpl{
+ z_stream_s* zstream;
+ InputStream* input;
+protected:
+ int32_t fillBuffer(signed char* start, int32_t space){
+ if (zstream == 0) return -1;
+ // make sure there is data to decompress
+ if (zstream->avail_in==0) {
+ // read data from the input stream
+ const signed char* inStart;
+ int32_t nread = input->read(inStart, 1, 0);
+ if (nread < 1) {
+ zstream->avail_in = 0; //bail...
+ if (deflate(zstream, Z_FINISH) != Z_STREAM_END) {
+ _CLTHROWA(CL_ERR_IO, "deflate should report Z_STREAM_END\n");
+ }
+ int32_t nwritten = space - zstream->avail_out;
+ dealloc();
+ return nwritten;
+ }
+ zstream->next_in = (Bytef*)inStart;
+ zstream->avail_in = nread;
+ }
+ // make sure we can write into the buffer
+ zstream->avail_out = space;
+ zstream->next_out = (Bytef*)start;
-GZipCompressInputStream::GZipCompressInputStream(StreamBase<char>* input, int level) {
- // initialize values that signal state
- status = Ok;
- zstream = 0;
- if ( level < 0 || level > 9 )
- level = Z_DEFAULT_COMPRESSION;
+ int r = deflate(zstream, Z_NO_FLUSH);
+ // inform the buffer of the number of bytes that was read
+ int32_t nwritten = space - zstream->avail_out;
+ switch (r) {
+ case Z_NEED_DICT:
+ _CLTHROWA(CL_ERR_IO, "Z_NEED_DICT while inflating stream.");
+ break;
+ case Z_DATA_ERROR:
+ _CLTHROWA(CL_ERR_IO, "Z_DATA_ERROR while inflating stream.");
+ break;
+ case Z_MEM_ERROR:
+ _CLTHROWA(CL_ERR_IO, "Z_MEM_ERROR while inflating stream.");
+ break;
+ }
+ return nwritten;
+ }
+public:
+ void dealloc(){
+ if (zstream) {
+ deflateEnd(zstream);
+ free(zstream);
+ zstream = 0;
+ }
+ }
+ void _setMinBufSize(int buf){
+ this->setMinBufSize(buf);
+ }
+ Internal(InputStream* input, int level){
+ if ( level < 0 || level > 9 )
+ level = Z_DEFAULT_COMPRESSION;
- this->input = input;
+ this->input = input;
- // initialize the z_stream
- zstream = (z_stream_s*)malloc(sizeof(z_stream_s));
- zstream->zalloc = Z_NULL;
- zstream->zfree = Z_NULL;
- zstream->opaque = Z_NULL;
- zstream->avail_in = 0;
-
- // initialize for writing gzip streams
- int r = deflateInit(zstream, level);
- if (r != Z_OK) {
- error = "Error initializing GZipCompressInputStream.";
- dealloc();
- status = Error;
- return;
- }
+ // initialize the z_stream
+ zstream = (z_stream_s*)malloc(sizeof(z_stream_s));
+ zstream->zalloc = Z_NULL;
+ zstream->zfree = Z_NULL;
+ zstream->opaque = Z_NULL;
+ zstream->avail_in = 0;
+
+ // initialize for writing gzip streams
+ int r = deflateInit(zstream, level);
+ if (r != Z_OK) {
+ dealloc();
+ _CLTHROWA(CL_ERR_IO, "Error initializing GZipCompressInputStream.");
+ }
- // signal that we need to read into the buffer
- zstream->avail_out = 1;
+ // signal that we need to read into the buffer
+ zstream->avail_out = 1;
+ }
- // initialize the buffer
- mark(262144);
+ ~Internal(){
+ dealloc();
+ }
+};
+
+
+GZipCompressInputStream::GZipCompressInputStream ( InputStream* input, int level)
+{
+ internal = new Internal(input,level);
}
-GZipCompressInputStream::~GZipCompressInputStream() {
- dealloc();
+size_t GZipCompressInputStream::size(){
+ return internal->size();
}
-void
-GZipCompressInputStream::dealloc() {
- if (zstream) {
- deflateEnd(zstream);
- free(zstream);
- zstream = 0;
- }
+GZipCompressInputStream::~GZipCompressInputStream ()
+{
+ delete internal;
}
-void
-GZipCompressInputStream::readFromStream() {
- // read data from the input stream
- const char* inStart;
- int32_t nread;
- nread = input->read(inStart, 1, 0);
- if (nread < -1) {
- status = Error;
- error = input->getError();
- } else if (nread < 1) {
- zstream->avail_in = 0; //bail...
- } else {
- zstream->next_in = (Bytef*)inStart;
- zstream->avail_in = nread;
- }
+int32_t GZipCompressInputStream::read(const signed char*& start, int32_t min, int32_t max){
+ return internal->read(start,min,max);
}
-int32_t
-GZipCompressInputStream::fillBuffer(char* start, int32_t space) {
- if (zstream == 0) return -1;
- // make sure there is data to decompress
- if (zstream->avail_in==0) {
- readFromStream();
- if (status == Error) {
- // no data was read
- return -1;
- }else if ( zstream->avail_in == 0 ){
- if (deflate(zstream, Z_FINISH) != Z_STREAM_END) {
- fprintf(stderr, "deflate should report Z_STREAM_END\n");
- exit(1);
- }
- int32_t nwritten = space - zstream->avail_out;
- dealloc();
- return nwritten;
- }
- }
- // make sure we can write into the buffer
- zstream->avail_out = space;
- zstream->next_out = (Bytef*)start;
+int64_t GZipCompressInputStream::position(){
+ return internal->position();
+}
+int64_t GZipCompressInputStream::reset(int64_t to){
+ return internal->reset(to);
+}
+int64_t GZipCompressInputStream::skip(int64_t ntoskip){
+ return internal->skip(ntoskip);
+}
+void GZipCompressInputStream::setMinBufSize(int32_t minbufsize){
+ internal->_setMinBufSize(minbufsize);
+}
- int r = deflate(zstream, Z_NO_FLUSH);
- // inform the buffer of the number of bytes that was read
- int32_t nwritten = space - zstream->avail_out;
- switch (r) {
- case Z_NEED_DICT:
- error = "Z_NEED_DICT while inflating stream.";
- status = Error;
- break;
- case Z_DATA_ERROR:
- error = "Z_DATA_ERROR while inflating stream.";
- status = Error;
- break;
- case Z_MEM_ERROR:
- error = "Z_MEM_ERROR while inflating stream.";
- status = Error;
- break;
- }
- return nwritten;
-}
+CL_NS_END
Modified: branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipcompressstream.h
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/gzipcompressstream.h 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipcompressstream.h 2009-03-30 21:34:13 UTC (rev 2976)
@@ -17,29 +17,31 @@
* the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
-#ifndef GZIPCOMPRESSSTREAM_H
-#define GZIPCOMPRESSSTREAM_H
+#ifndef CLUENE_UTIL_GZIPCOMPRESSSTREAM_H
+#define CLUENE_UTIL_GZIPCOMPRESSSTREAM_H
-#include "bufferedstream.h"
+#include "CLucene/util/CLStreams.h"
struct z_stream_s;
-namespace jstreams {
+CL_NS_DEF(util)
-class GZipCompressInputStream : public BufferedInputStream<char> {
+class GZipCompressInputStream : public InputStream{
private:
- z_stream_s* zstream;
- StreamBase<char>* input;
-
- void dealloc();
- void readFromStream();
- void compressFromStream();
+ class Internal;
+ Internal* internal;
public:
- explicit GZipCompressInputStream(StreamBase<char>* input, int level=-1);
- ~GZipCompressInputStream();
- int32_t fillBuffer(char* start, int32_t space);
+ LUCENE_STATIC_CONSTANT(int32_t, DEFAULT_BUFFER_SIZE=4096);
+ explicit GZipCompressInputStream(InputStream* input, int level=-1);
+ virtual ~GZipCompressInputStream();
+
+ int32_t read(const signed char*& start, int32_t min, int32_t max);
+ int64_t position();
+ int64_t reset(int64_t);
+ int64_t skip(int64_t ntoskip);
+ size_t size();
+ void setMinBufSize(int32_t minbufsize);
};
-} // end namespace jstreams
-
+CL_NS_END
#endif
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipinputstream.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/gzipinputstream.cpp 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipinputstream.cpp 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,145 +0,0 @@
-/*------------------------------------------------------------------------------
-* Copyright (C) 2003-2006 Jos van den Oever
-*
-* Distributable under the terms of either the Apache License (Version 2.0) or
-* the GNU Lesser General Public License, as specified in the COPYING file.
-------------------------------------------------------------------------------*/
-#include "CLucene/jstreams/jstreamsconfig.h"
-#include "gzipinputstream.h"
-#include <zlib.h>
-using namespace jstreams;
-
-GZipInputStream::GZipInputStream(StreamBase<char>* input, ZipFormat format) {
- // initialize values that signal state
- status = Ok;
- zstream = 0;
-
- this->input = input;
-
- // check first bytes of stream before allocating buffer
- if (format == GZIPFORMAT && !checkMagic()) {
- error = "Magic bytes are wrong.";
- status = Error;
- return;
- }
-
- // initialize the z_stream
- zstream = (z_stream_s*)malloc(sizeof(z_stream_s));
- zstream->zalloc = Z_NULL;
- zstream->zfree = Z_NULL;
- zstream->opaque = Z_NULL;
- zstream->avail_in = 0;
- zstream->next_in = Z_NULL;
- // initialize for reading gzip streams
- // for reading libz streams, you need inflateInit(zstream)
- int r;
- switch(format) {
- case ZLIBFORMAT:
- r = inflateInit(zstream);
- break;
- case GZIPFORMAT:
- r = inflateInit2(zstream, 15+16);
- break;
- case ZIPFORMAT:
- default:
- r = inflateInit2(zstream, -MAX_WBITS);
- break;
- }
- if (r != Z_OK) {
- error = "Error initializing GZipInputStream.";
- dealloc();
- status = Error;
- return;
- }
-
- // signal that we need to read into the buffer
- zstream->avail_out = 1;
-
- // initialize the buffer
- mark(262144);
-}
-GZipInputStream::~GZipInputStream() {
- dealloc();
-}
-void
-GZipInputStream::dealloc() {
- if (zstream) {
- inflateEnd(zstream);
- free(zstream);
- zstream = 0;
- }
-}
-bool
-GZipInputStream::checkMagic() {
- const unsigned char* buf;
- const char* begin;
- int32_t nread;
-
- int64_t pos = input->getPosition();
- nread = input->read(begin, 2, 2);
- input->reset(pos);
- if (nread != 2) {
- return false;
- }
-
- buf = (const unsigned char*)begin;
- return buf[0] == 0x1f && buf[1] == 0x8b;
-}
-void
-GZipInputStream::readFromStream() {
- // read data from the input stream
- const char* inStart;
- int32_t nread;
- nread = input->read(inStart, 1, 0);
- if (nread < -1) {
- status = Error;
- error = input->getError();
- } else if (nread < 1) {
- status = Error;
- error = "unexpected end of stream";
- } else {
- zstream->next_in = (Bytef*)inStart;
- zstream->avail_in = nread;
- }
-}
-int32_t
-GZipInputStream::fillBuffer(char* start, int32_t space) {
- if (zstream == 0) return -1;
- // make sure there is data to decompress
- if (zstream->avail_out) {
- readFromStream();
- if (status == Error) {
- // no data was read
- return -1;
- }
- }
- // make sure we can write into the buffer
- zstream->avail_out = space;
- zstream->next_out = (Bytef*)start;
- // decompress
- int r = inflate(zstream, Z_SYNC_FLUSH);
- // inform the buffer of the number of bytes that was read
- int32_t nwritten = space - zstream->avail_out;
- switch (r) {
- case Z_NEED_DICT:
- error = "Z_NEED_DICT while inflating stream.";
- status = Error;
- break;
- case Z_DATA_ERROR:
- error = "Z_DATA_ERROR while inflating stream.";
- status = Error;
- break;
- case Z_MEM_ERROR:
- error = "Z_MEM_ERROR while inflating stream.";
- status = Error;
- break;
- case Z_STREAM_END:
- if (zstream->avail_in) {
- input->reset(input->getPosition()-zstream->avail_in);
- }
- // we are finished decompressing,
- // (but this stream is not yet finished)
- dealloc();
- }
- return nwritten;
-}
Copied: branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipinputstream.cpp (from rev 2949, branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/gzipinputstream.cpp)
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipinputstream.cpp (rev 0)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipinputstream.cpp 2009-03-30 21:34:13 UTC (rev 2976)
@@ -0,0 +1,184 @@
+/*------------------------------------------------------------------------------
+* Copyright (C) 2003-2006 Jos van den Oever
+*
+* Distributable under the terms of either the Apache License (Version 2.0) or
+* the GNU Lesser General Public License, as specified in the COPYING file.
+------------------------------------------------------------------------------*/
+#include "CLucene/_ApiHeader.h"
+#include "gzipinputstream.h"
+#include "CLucene/util/_bufferedstream.h"
+#include <zlib.h>
+
+CL_NS_DEF(util)
+
+
+class GZipInputStream::Internal{
+public:
+ class JStreamsBuffer: public BufferedInputStreamImpl{
+ z_stream_s* zstream;
+ BufferedInputStream* input;
+ protected:
+
+ int32_t fillBuffer(signed char* start, int32_t space) {
+ if (zstream == 0) return -1;
+ // make sure there is data to decompress
+ if (zstream->avail_out) {
+ // read data from the input stream
+ const signed char* inStart;
+ int32_t nread = input->read(inStart, 1, 0);
+ if (nread < 1) {
+ _CLTHROWA(CL_ERR_IO, "unexpected end of stream");
+ } else {
+ zstream->next_in = (Bytef*)inStart;
+ zstream->avail_in = nread;
+ }
+ }
+ // make sure we can write into the buffer
+ zstream->avail_out = space;
+ zstream->next_out = (Bytef*)start;
+ // decompress
+ int r = inflate(zstream, Z_SYNC_FLUSH);
+ // inform the buffer of the number of bytes that was read
+ int32_t nwritten = space - zstream->avail_out;
+ switch (r) {
+ case Z_NEED_DICT:
+ _CLTHROWA(CL_ERR_IO, "Z_NEED_DICT while inflating stream.");
+ break;
+ case Z_DATA_ERROR:
+ _CLTHROWA(CL_ERR_IO, "Z_DATA_ERROR while inflating stream.");
+ break;
+ case Z_MEM_ERROR:
+ _CLTHROWA(CL_ERR_IO, "Z_MEM_ERROR while inflating stream.");
+ break;
+ case Z_STREAM_END:
+ if (zstream->avail_in) {
+ input->reset(input->getPosition()-zstream->avail_in);
+ }
+ // we are finished decompressing,
+ // (but this stream is not yet finished)
+ dealloc();
+ }
+ return nwritten;
+ }
+ void dealloc() {
+ if (zstream) {
+ inflateEnd(zstream);
+ free(zstream);
+ zstream = 0;
+ }
+ }
+ bool checkMagic() {
+ const unsigned char* buf;
+ const signed char* begin;
+ int32_t nread;
+
+ int64_t pos = input->getPosition();
+ nread = input->read(begin, 2, 2);
+ input->reset(pos);
+ if (nread != 2) {
+ return false;
+ }
+
+ buf = (const unsigned char*)begin;
+ return buf[0] == 0x1f && buf[1] == 0x8b;
+ }
+
+ public:
+ int encoding;
+
+ JStreamsBuffer(BufferedInputStream* input, GZipInputStream::ZipFormat format){
+ this->input = input;
+
+ // check first bytes of stream before allocating buffer
+ if (format == GZipInputStream::GZIPFORMAT && !checkMagic()) {
+ _CLTHROWA(CL_ERR_IO, "Magic bytes are wrong.");
+ }
+
+ // initialize the z_stream
+ zstream = (z_stream_s*)malloc(sizeof(z_stream_s));
+ zstream->zalloc = Z_NULL;
+ zstream->zfree = Z_NULL;
+ zstream->opaque = Z_NULL;
+ zstream->avail_in = 0;
+ zstream->next_in = Z_NULL;
+ // initialize for reading gzip streams
+ // for reading libz streams, you need inflateInit(zstream)
+ int r;
+ switch(format) {
+ case GZipInputStream::ZLIBFORMAT:
+ r = inflateInit(zstream);
+ break;
+ case GZipInputStream::GZIPFORMAT:
+ r = inflateInit2(zstream, 15+16);
+ break;
+ case GZipInputStream::ZIPFORMAT:
+ default:
+ r = inflateInit2(zstream, -MAX_WBITS);
+ break;
+ }
+ if (r != Z_OK) {
+ dealloc();
+ _CLTHROWA(CL_ERR_IO, "Error initializing GZipInputStream.");
+ }
+
+ // signal that we need to read into the buffer
+ zstream->avail_out = 1;
+ }
+ void _setMinBufSize(int32_t bufsize){
+ this->setMinBufSize(bufsize);
+ }
+
+ ~JStreamsBuffer(){
+ dealloc();
+ }
+ };
+
+ JStreamsBuffer* jsbuffer;
+
+ Internal(BufferedInputStream* input, GZipInputStream::ZipFormat format){
+ jsbuffer = new JStreamsBuffer(input, format);
+ }
+ ~Internal(){
+ delete jsbuffer;
+ }
+};
+
+GZipInputStream::GZipInputStream(InputStream* input, ZipFormat format) {
+ internal = new Internal(_CLNEW FilteredBufferedInputStream(input, false), format);
+}
+GZipInputStream::GZipInputStream(BufferedInputStream* input, ZipFormat format) {
+ internal = new Internal(input, format);
+}
+
+GZipInputStream::~GZipInputStream() {
+ delete internal;
+}
+
+
+int32_t GZipInputStream::read(const signed char*& start, int32_t min, int32_t max){
+ return internal->jsbuffer->read(start,min,max);
+}
+int32_t GZipInputStream::read(const unsigned char*& _start, int32_t min, int32_t max){
+ const signed char* start = 0;
+ int32_t ret = internal->jsbuffer->read(start,min,max);
+ _start = (const unsigned char*)start;
+ return ret;
+}
+int64_t GZipInputStream::position(){
+ return internal->jsbuffer->position();
+}
+int64_t GZipInputStream::reset(int64_t to){
+ return internal->jsbuffer->reset(to);
+}
+int64_t GZipInputStream::skip(int64_t ntoskip){
+ return internal->jsbuffer->skip(ntoskip);
+}
+void GZipInputStream::setMinBufSize(int32_t minbufsize){
+ internal->jsbuffer->_setMinBufSize(minbufsize);
+}
+size_t GZipInputStream::size(){
+ return internal->jsbuffer->size();
+}
+
+
+CL_NS_END
Property changes on: branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipinputstream.cpp
___________________________________________________________________
Added: svn:mergeinfo
+
Modified: branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipinputstream.h
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/gzipinputstream.h 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/gzipinputstream.h 2009-03-30 21:34:13 UTC (rev 2976)
@@ -4,32 +4,36 @@
* Distributable under the terms of either the Apache License (Version 2.0) or
* the GNU Lesser General Public License, as specified in the COPYING file.
------------------------------------------------------------------------------*/
-#ifndef GZIPINPUTSTREAM_H
-#define GZIPINPUTSTREAM_H
+#ifndef CLUENE_UTIL_GZIPINPUTSTREAM_H
+#define CLUENE_UTIL_GZIPINPUTSTREAM_H
-#include "bufferedstream.h"
+#include "CLucene/util/CLStreams.h"
struct z_stream_s;
-namespace jstreams {
+CL_NS_DEF(util)
-class GZipInputStream : public BufferedInputStream<char> {
+class GZipInputStream : public CL_NS(util)::BufferedInputStream {
+public:
+ enum ZipFormat { ZLIBFORMAT, GZIPFORMAT, ZIPFORMAT};
private:
- z_stream_s* zstream;
- StreamBase<char>* input;
-
- void dealloc();
- void readFromStream();
- void decompressFromStream();
- bool checkMagic();
+ class Internal;
+ Internal* internal;
public:
- enum ZipFormat { ZLIBFORMAT, GZIPFORMAT, ZIPFORMAT};
- explicit GZipInputStream(StreamBase<char>* input,
+ explicit GZipInputStream(BufferedInputStream* input,
ZipFormat format=GZIPFORMAT);
+ explicit GZipInputStream(InputStream* input,
+ ZipFormat format=GZIPFORMAT);
~GZipInputStream();
- int32_t fillBuffer(char* start, int32_t space);
+
+ int32_t read(const signed char*& start, int32_t min, int32_t max);
+ int32_t read(const unsigned char*& start, int32_t min, int32_t max);
+ int64_t position();
+ int64_t reset(int64_t);
+ int64_t skip(int64_t ntoskip);
+ size_t size();
+ void setMinBufSize(int32_t minbufsize);
};
-} // end namespace jstreams
-
+CL_NS_END
#endif
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/inputstream.h
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/inputstream.h 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/inputstream.h 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,18 +0,0 @@
-/*------------------------------------------------------------------------------
-* Copyright (C) 2003-2006 Jos van den Oever
-*
-* Distributable under the terms of either the Apache License (Version 2.0) or
-* the GNU Lesser General Public License, as specified in the COPYING file.
-------------------------------------------------------------------------------*/
-#ifndef INPUTSTREAM_H
-#define INPUTSTREAM_H
-
-#include "streambase.h"
-
-namespace jstreams {
-
-typedef StreamBase<char> InputStream;
-
-} // end namespace jstreams
-
-#endif
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/inputstreambuffer.h
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/inputstreambuffer.h 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/inputstreambuffer.h 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,102 +0,0 @@
-/*------------------------------------------------------------------------------
-* Copyright (C) 2003-2006 Jos van den Oever
-*
-* Distributable under the terms of either the Apache License (Version 2.0) or
-* the GNU Lesser General Public License, as specified in the COPYING file.
-------------------------------------------------------------------------------*/
-#ifndef INPUTSTREAMBUFFER_H
-#define INPUTSTREAMBUFFER_H
-
-namespace jstreams {
-
-template <class T>
-class InputStreamBuffer {
-private:
-public:
- T* start;
- int32_t size;
- T* readPos;
- int32_t avail;
-
- InputStreamBuffer();
- ~InputStreamBuffer();
- void setSize(int32_t size);
- int32_t read(const T*& start, int32_t max=0);
-
- /**
- * This function prepares the buffer for a new write.
- * returns the number of available places.
- **/
- int32_t makeSpace(int32_t needed);
-};
-
-template <class T>
-InputStreamBuffer<T>::InputStreamBuffer() {
- readPos = start = 0;
- size = avail = 0;
-}
-template <class T>
-InputStreamBuffer<T>::~InputStreamBuffer() {
- free(start);
-}
-template <class T>
-void
-InputStreamBuffer<T>::setSize(int32_t size) {
- // store pointer information
- int32_t offset = readPos - start;
-
- // allocate memory in the buffer
- start = (T*)realloc(start, size*sizeof(T));
- this->size = size;
-
- // restore pointer information
- readPos = start + offset;
-}
-template <class T>
-int32_t
-InputStreamBuffer<T>::makeSpace(int32_t needed) {
- // determine how much space is available for writing
- int32_t space = size - (readPos - start) - avail;
- if (space >= needed) {
- // there's enough space
- return space;
- }
-
- if (avail) {
- if (readPos != start) {
-// printf("moving\n");
- // move data to the start of the buffer
- memmove(start, readPos, avail*sizeof(T));
- space += readPos - start;
- readPos = start;
- }
- } else {
- // we may start writing at the start of the buffer
- readPos = start;
- space = size;
- }
- if (space >= needed) {
- // there's enough space now
- return space;
- }
-
- // still not enough space, we have to allocate more
-// printf("resize %i %i %i %i %i\n", avail, needed, space, size + needed - space, size);
- setSize(size + needed - space);
- return needed;
-}
-template <class T>
-int32_t
-InputStreamBuffer<T>::read(const T*& start, int32_t max) {
- start = readPos;
- if (max <= 0 || max > avail) {
- max = avail;
- }
- readPos += max;
- avail -= max;
- return max;
-}
-
-} // end namespace jstreams
-
-#endif
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/inputstreamreader.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/inputstreamreader.cpp 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/inputstreamreader.cpp 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,132 +0,0 @@
-/*------------------------------------------------------------------------------
-* Copyright (C) 2003-2006 Jos van den Oever
-*
-* Distributable under the terms of either the Apache License (Version 2.0) or
-* the GNU Lesser General Public License, as specified in the COPYING file.
-------------------------------------------------------------------------------*/
-#include "CLucene/jstreams/jstreamsconfig.h"
-#include "inputstreamreader.h"
-#include "CLucene/jstreams/jstreamsconfig.h"
-#include <cerrno>
-using namespace jstreams;
-
-#ifndef ICONV_CONST
- //we try to guess whether the iconv function requires
- //a const char. We have no way of automatically figuring
- //this out if we did not use autoconf, so we guess based
- //on certain parameters:
- #ifdef _LIBICONV_H
- #define ICONV_CONST const
- #else
- #define ICONV_CONST
- #endif
-#endif
-
-InputStreamReader::InputStreamReader(StreamBase<char>* i, const char* enc) {
- status = Ok;
- finishedDecoding = false;
- input = i;
- if (enc == 0) enc = "UTF-8";
-#ifdef _LIBICONV_H
- if (sizeof(wchar_t) == 4) {
- converter = iconv_open("UCS-4-INTERNAL", enc);
- } if (sizeof(wchar_t) == 2) {
- converter = iconv_open("UCS-2-INTERNAL", enc);
-#else
- if (sizeof(wchar_t) > 1) {
- converter = iconv_open("WCHAR_T", enc);
-#endif
- } else {
- converter = iconv_open("ASCII", enc);
- }
-
- // check if the converter is valid
- if (converter == (iconv_t) -1) {
- error = "conversion from '";
- error += enc;
- error += "' not available.";
- status = Error;
- return;
- }
- charbuf.setSize(262);
- //mark(262);
- charsLeft = 0;
-}
-InputStreamReader::~InputStreamReader() {
- if (converter != (iconv_t) -1) {
- iconv_close(converter);
- }
-}
-int32_t
-InputStreamReader::decode(wchar_t* start, int32_t space) {
- // decode from charbuf
- ICONV_CONST char *inbuf = charbuf.readPos;
- size_t inbytesleft = charbuf.avail;
- size_t outbytesleft = sizeof(wchar_t)*space;
- char *outbuf = (char*)start;
- size_t r = iconv(converter, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
- int32_t nwritten;
- if (r == (size_t)-1) {
- switch (errno) {
- case EILSEQ: //invalid multibyte sequence
- error = "Invalid multibyte sequence.";
- status = Error;
- return -1;
- case EINVAL: // last character is incomplete
- // move from inbuf to the end to the start of
- // the buffer
- memmove(charbuf.start, inbuf, inbytesleft);
- charbuf.readPos = charbuf.start;
- charbuf.avail = inbytesleft;
- nwritten = ((wchar_t*)outbuf) - start;
- break;
- case E2BIG: // output buffer is full
- charbuf.readPos += charbuf.avail - inbytesleft;
- charbuf.avail = inbytesleft;
- nwritten = space;
- break;
- default:
- printf("InputStreamReader::error %d\n", errno);
- }
- } else { //input sequence was completely converted
- charbuf.readPos = charbuf.start;
- charbuf.avail = 0;
- nwritten = ((wchar_t*)outbuf) - start;
- if (input == 0) {
- finishedDecoding = true;
- }
- }
- return nwritten;
-}
-int32_t
-InputStreamReader::fillBuffer(wchar_t* start, int32_t space) {
- // fill up charbuf
- if (input && charbuf.readPos == charbuf.start) {
- const char *begin;
- int32_t numRead;
- numRead = input->read(begin, 1, charbuf.size - charbuf.avail);
- //printf("filled up charbuf\n");
- if (numRead < -1) {
- error = input->getError();
- status = Error;
- input = 0;
- return numRead;
- }
- if (numRead < 1) {
- // signal end of input buffer
- input = 0;
- if (charbuf.avail) {
- error = "stream ends on incomplete character";
- status = Error;
- }
- return -1;
- }
- // copy data into other buffer
- memmove(charbuf.start + charbuf.avail, begin, numRead);
- charbuf.avail = numRead + charbuf.avail;
- }
- // decode
- int32_t n = decode(start, space);
- //printf("decoded %i\n", n);
- return n;
-}
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/inputstreamreader.h
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/inputstreamreader.h 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/inputstreamreader.h 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,39 +0,0 @@
-/*------------------------------------------------------------------------------
-* Copyright (C) 2003-2006 Jos van den Oever
-*
-* Distributable under the terms of either the Apache License (Version 2.0) or
-* the GNU Lesser General Public License, as specified in the COPYING file.
-------------------------------------------------------------------------------*/
-#ifndef INPUTSTREAMREADER_H
-#define INPUTSTREAMREADER_H
-
-#include <string>
-#include <iconv.h>
-#include "bufferedstream.h"
-
-namespace jstreams {
-
-/**
- * Info on conversions:
-http://www.gnu.org/software/libc/manual/html_node/iconv-Examples.html
-http://tangentsoft.net/mysql++/doc/userman/html/unicode.html
- **/
-class InputStreamReader : public BufferedInputStream<wchar_t> {
-private:
- iconv_t converter;
- bool finishedDecoding;
- StreamBase<char>* input;
- int32_t charsLeft;
-
- InputStreamBuffer<char> charbuf;
- void readFromStream();
- int32_t decode(wchar_t* start, int32_t space);
-public:
- explicit InputStreamReader(StreamBase<char> *i, const char *enc=0);
- ~InputStreamReader();
- int32_t fillBuffer(wchar_t* start, int32_t space);
-};
-
-} // end namespace jstreams
-
-#endif
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/jstreamsconfig.h.cmake
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/jstreamsconfig.h.cmake 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/jstreamsconfig.h.cmake 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,12 +0,0 @@
-#ifndef JSTREAMS_JSTREAMCONFIG_H
-#define JSTREAMS_JSTREAMCONFIG_H
-
-#include "CLucene/_ApiHeader.h"
-
-#if @ICONV_SECOND_ARGUMENT_IS_CONST@
- #define _CL_ICONV_CONST const
-#else
- #define _CL_ICONV_CONST
-#endif
-
-#endif
Deleted: branches/lucene2_3_2/src/contribs-lib/CLucene/util/streambase.h
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/jstreams/streambase.h 2009-01-10 17:17:55 UTC (rev 2949)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/util/streambase.h 2009-03-30 21:34:13 UTC (rev 2976)
@@ -1,124 +0,0 @@
-/*------------------------------------------------------------------------------
-* Copyright (C) 2003-2006 Jos van den Oever
-*
-* Distributable under the terms of either the Apache License (Version 2.0) or
-* the GNU Lesser General Public License, as specified in the COPYING file.
-------------------------------------------------------------------------------*/
-#ifndef STREAMBASE_H
-#define STREAMBASE_H
-
-#include <string>
-
-namespace jstreams {
-
-enum StreamStatus { Ok, Eof, Error };
-
-/**
- * @short Base class for stream read access to many different file types.
- *
- * This class is based on the interface java.io.InputStream. It allows
- * for uniform access to streamed resources.
- * The main difference with the java equivalent is a performance improvement.
- * When reading data, data is not copied into a buffer provided by the caller,
- * but a pointer to the read data is provided. This makes this interface
- * especially useful for deriving from it and implementing filterers or
- * transformers.
- */
-// java mapping: long=int64, int=int32, byte=uint8_t
-template <class T>
-class StreamBase {
-protected:
- int64_t size;
- int64_t position;
- std::string error;
- StreamStatus status;
-public:
- StreamBase() :size(-1), position(0), status(Ok){ }
- virtual ~StreamBase(){}
- /**
- * @brief Return a string representation of the last error.
- * If no error has occurred, an empty string is returned.
- **/
- const char* getError() const { return error.c_str(); }
- StreamStatus getStatus() const { return status; }
- /**
- * @brief Get the current position in the stream.
- * The value obtained from this function can be used to reset the stream.
- **/
- int64_t getPosition() const { return position; }
- /**
- * @brief Return the size of the stream.
- * If the size of the stream is unknown, -1
- * is returned. If the end of the stream has been reached the size is
- * always known.
- **/
- int64_t getSize() const { return size; }
- /**
- * @brief Reads characters from the stream and sets \a start to
- * the first character that was read.
- *
- * If @p ntoread is @c 0, then at least one character will be read.
- *
- * @param start Pointer passed by reference that will be set to point to
- * the retrieved array of characters. If the end of the stream
- * is encountered or an error occurs, the value of @p start
- * is undefined.
- * @param ntoread The number of characters to read from the stream. If
- * @p is @c 0 the stream reads at least 1 character.
- * @return the number of characters that were read. If -1 is returned, the
- * end of the stream has been reached. If -2 is returned, an error
- * has occurred.
- **/
- virtual int32_t read(const T*& start, int32_t min, int32_t max) = 0;
- /**
- * Skip @param ntoskip bytes. Unless an error occurs or the end of file is
- * encountered, this amount of bytes is skipped.
- * This function returns new position in the stream.
- **/
- virtual int64_t skip(int64_t ntoskip);
- /**
- * @brief Repositions this stream to gi...
[truncated message content] |
|
From: <ust...@us...> - 2009-03-30 21:07:55
|
Revision: 2975
http://clucene.svn.sourceforge.net/clucene/?rev=2975&view=rev
Author: ustramooner
Date: 2009-03-30 21:07:45 +0000 (Mon, 30 Mar 2009)
Log Message:
-----------
deprecate functions
Modified Paths:
--------------
branches/lucene2_3_2/src/shared/CLucene/util/Misc.h
Modified: branches/lucene2_3_2/src/shared/CLucene/util/Misc.h
===================================================================
--- branches/lucene2_3_2/src/shared/CLucene/util/Misc.h 2009-03-30 21:06:57 UTC (rev 2974)
+++ branches/lucene2_3_2/src/shared/CLucene/util/Misc.h 2009-03-30 21:07:45 UTC (rev 2975)
@@ -49,8 +49,8 @@
static TCHAR* stringTrim(TCHAR* s);
static TCHAR* wordTrim(TCHAR* s);
- static char* longToBase( int64_t value, int32_t base );
- static int64_t base36ToLong( const char* value );
+ _CL_DEPRECATED(_i64tot) static char* longToBase( int64_t value, int32_t base );
+ _CL_DEPRECATED(_tcstoi64) static int64_t base36ToLong( const char* value );
#ifdef _UCS2
static size_t whashCode(const wchar_t* str);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-30 21:07:07
|
Revision: 2974
http://clucene.svn.sourceforge.net/clucene/?rev=2974&view=rev
Author: ustramooner
Date: 2009-03-30 21:06:57 +0000 (Mon, 30 Mar 2009)
Log Message:
-----------
fix memleak
Modified Paths:
--------------
branches/lucene2_3_2/src/contribs-lib/CLucene/snowball/Snowball.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/snowball/SnowballAnalyzer.h
Modified: branches/lucene2_3_2/src/contribs-lib/CLucene/snowball/Snowball.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/snowball/Snowball.cpp 2009-03-30 20:57:49 UTC (rev 2973)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/snowball/Snowball.cpp 2009-03-30 21:06:57 UTC (rev 2974)
@@ -2,6 +2,7 @@
#include "SnowballAnalyzer.h"
#include "SnowballFilter.h"
#include "CLucene/util/Misc.h"
+#include "CLucene/util/CLStreams.h"
#include "CLucene/analysis/Analyzers.h"
#include "CLucene/analysis/standard/StandardTokenizer.h"
#include "CLucene/analysis/standard/StandardFilter.h"
@@ -33,17 +34,22 @@
StopFilter::fillStopTable(stopSet,stopWords);
}
+ TokenStream* SnowballAnalyzer::tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader) {
+ return this->tokenStream(fieldName,reader,false);
+ }
+
/** Constructs a {@link StandardTokenizer} filtered by a {@link
StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}. */
- TokenStream* SnowballAnalyzer::tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader) {
- TokenStream* result = NULL;
- BufferedReader* bufferedReader = reader->__asBufferedReader();
- if ( bufferedReader == NULL )
- result = _CLNEW StandardTokenizer( _CLNEW FilteredBufferedReader(reader, false), true );
- else
- result = _CLNEW StandardTokenizer(bufferedReader);
+ TokenStream* SnowballAnalyzer::tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader, bool deleteReader) {
+ BufferedReader* bufferedReader = reader->__asBufferedReader();
+ TokenStream* result;
- result = _CLNEW StandardFilter(result, true);
+ if ( bufferedReader == NULL )
+ result = _CLNEW StandardTokenizer( _CLNEW FilteredBufferedReader(reader, deleteReader), true );
+ else
+ result = _CLNEW StandardTokenizer(bufferedReader, deleteReader);
+
+ result = _CLNEW StandardFilter(result, true);
result = _CLNEW CL_NS(analysis)::LowerCaseFilter(result, true);
if (stopSet != NULL)
result = _CLNEW CL_NS(analysis)::StopFilter(result, true, stopSet);
Modified: branches/lucene2_3_2/src/contribs-lib/CLucene/snowball/SnowballAnalyzer.h
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/snowball/SnowballAnalyzer.h 2009-03-30 20:57:49 UTC (rev 2973)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/snowball/SnowballAnalyzer.h 2009-03-30 21:06:57 UTC (rev 2974)
@@ -3,6 +3,7 @@
#include "CLucene/analysis/AnalysisHeader.h"
+CL_CLASS_DEF(util,BufferedReader)
CL_NS_DEF2(analysis,snowball)
/** Filters {@link StandardTokenizer} with {@link StandardFilter}, {@link
@@ -29,6 +30,7 @@
/** Constructs a {@link StandardTokenizer} filtered by a {@link
StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}. */
TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
+ TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader, bool deleteReader);
};
CL_NS_END2
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-30 20:57:59
|
Revision: 2973
http://clucene.svn.sourceforge.net/clucene/?rev=2973&view=rev
Author: ustramooner
Date: 2009-03-30 20:57:49 +0000 (Mon, 30 Mar 2009)
Log Message:
-----------
move away the jstreams into the standard util directory. we're going to support jstreams through a filter class instead.... (Soon!)
Modified Paths:
--------------
branches/lucene2_3_2/src/core/CLucene/util/CLStreams.h
branches/lucene2_3_2/src/core/CLucene/util/Reader.cpp
branches/lucene2_3_2/src/core/CLucene/util/_bufferedstream.h
branches/lucene2_3_2/src/core/CLucene/util/_streambase.h
branches/lucene2_3_2/src/core/CLucene/util/_streambuffer.h
Modified: branches/lucene2_3_2/src/core/CLucene/util/CLStreams.h
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/util/CLStreams.h 2009-03-30 20:54:55 UTC (rev 2972)
+++ branches/lucene2_3_2/src/core/CLucene/util/CLStreams.h 2009-03-30 20:57:49 UTC (rev 2973)
@@ -95,6 +95,7 @@
* The value obtained from this function can be used to reset the stream.
**/
virtual int64_t position() = 0;
+ int64_t getPosition(){ return this->position(); }
virtual size_t size() = 0;
};
@@ -175,7 +176,22 @@
void setMinBufSize(int32_t minbufsize);
};
+class CLUCENE_EXPORT FilteredBufferedInputStream: public BufferedInputStream{
+ class Internal;
+ Internal* internal;
+public:
+ FilteredBufferedInputStream(InputStream* input, bool deleteInput);
+ virtual ~FilteredBufferedInputStream();
+
+ int32_t read(const signed char*& start, int32_t min, int32_t max);
+ int64_t position();
+ int64_t reset(int64_t);
+ int64_t skip(int64_t ntoskip);
+ size_t size();
+ void setMinBufSize(int32_t minbufsize);
+};
+
class CLUCENE_EXPORT StringReader: public BufferedReader{
TCHAR* value;
bool ownValue;
@@ -185,11 +201,30 @@
public:
StringReader ( const TCHAR* value, const int32_t length = -1 );
StringReader ( TCHAR* value, const int32_t length, bool copyData = true );
- virtual ~StringReader();
+ virtual ~StringReader();
int32_t read(const TCHAR*& start, int32_t min, int32_t max);
int64_t position();
int64_t reset(int64_t);
+ int64_t skip(int64_t ntoskip);
+ void setMinBufSize(int32_t s);
+ size_t size();
+};
+class CLUCENE_EXPORT AStringReader: public BufferedInputStream{
+ signed char* value;
+ bool ownValue;
+ int64_t pos;
+protected:
+ size_t m_size;
+public:
+ AStringReader ( const char* value, const int32_t length = -1 );
+ AStringReader ( char* value, const int32_t length, bool copyData = true );
+ virtual ~AStringReader();
+
+ int32_t read(const signed char*& start, int32_t min, int32_t max);
+ int32_t read(const unsigned char*& start, int32_t min, int32_t max);
+ int64_t position();
+ int64_t reset(int64_t);
int64_t skip(int64_t ntoskip);
void setMinBufSize(int32_t s);
size_t size();
@@ -230,6 +265,7 @@
};
SimpleInputStreamReader();
+ SimpleInputStreamReader(InputStream *i, int encoding);
virtual ~SimpleInputStreamReader();
int32_t read(const TCHAR*& start, int32_t min, int32_t max);
@@ -255,4 +291,7 @@
};
CL_NS_END
+
+#define jstreams CL_NS(util)
+
#endif
Modified: branches/lucene2_3_2/src/core/CLucene/util/Reader.cpp
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/util/Reader.cpp 2009-03-30 20:54:55 UTC (rev 2972)
+++ branches/lucene2_3_2/src/core/CLucene/util/Reader.cpp 2009-03-30 20:57:49 UTC (rev 2973)
@@ -82,9 +82,75 @@
return s;
}
+
+
+
+AStringReader::AStringReader ( char* value, const int32_t length, bool copyData )
+{
+ this->m_size = length;
+ this->pos = 0;
+ if ( copyData ){
+ this->value = _CL_NEWARRAY(signed char, this->m_size);
+ strncpy((char*)this->value, value, this->m_size);
+ }else{
+ this->value = (signed char*)value;
+ }
+ this->ownValue = copyData;
+}
+
+AStringReader::AStringReader ( const char* value, const int32_t length ){
+ if ( length >= 0 )
+ this->m_size = length;
+ else
+ this->m_size = strlen(value);
+ this->pos = 0;
+ this->value = _CL_NEWARRAY(signed char, this->m_size);
+ strncpy((char*)this->value, value, this->m_size);
+ this->ownValue = true;
+}
+AStringReader::~AStringReader(){
+ if ( ownValue )
+ _CLDELETE_ARRAY(this->value);
+}
+
+size_t AStringReader::size(){
+ return m_size;
+}
+int32_t AStringReader::read(const signed char*& start, int32_t min, int32_t max){
+ if ( m_size == pos )
+ return -1;
+ start = this->value + pos;
+ int32_t r = (int32_t)cl_min(cl_max(min,max),m_size-pos);
+ pos += r;
+ return r;
+}
+int32_t AStringReader::read(const unsigned char*& start, int32_t min, int32_t max){
+ if ( m_size == pos )
+ return -1;
+ start = (unsigned char*)(this->value + pos);
+ int32_t r = (int32_t)cl_min(cl_max(min,max),m_size-pos);
+ pos += r;
+ return r;
+}
+int64_t AStringReader::position(){
+ return pos;
+}
+void AStringReader::setMinBufSize(int32_t s){
+}
+int64_t AStringReader::reset(int64_t pos){
+ if ( pos >= 0 && pos < this->m_size )
+ this->pos = pos;
+ return this->pos;
+}
+int64_t AStringReader::skip(int64_t ntoskip){
+ int64_t s = cl_min(ntoskip, m_size-pos);
+ this->pos += s;
+ return s;
+}
+
class FileInputStream::Internal{
public:
- class JStreamsBuffer: public jstreams::BufferedInputStream{
+ class JStreamsBuffer: public BufferedInputStreamImpl{
int32_t fhandle;
protected:
int32_t fillBuffer(signed char* start, int32_t space){
@@ -95,7 +161,7 @@
// check the file stream status
if (nwritten == -1 ) {
m_error = "Could not read from file";
- m_status = jstreams::Error;
+ m_status = CL_NS(util)::Error;
if ( fhandle > 0 ){
::_close(fhandle);
fhandle = 0;
@@ -146,7 +212,7 @@
else if ( err == EMFILE )
_CLTHROWA(CL_ERR_IO, "Too many open files");
else
- _CLTHROWA(CL_ERR_IO, "Could not open file");
+ _CLTHROWA(CL_ERR_IO, "Could not open file");
}
jsbuffer = new JStreamsBuffer(fhandle, buffersize);
@@ -215,7 +281,7 @@
class SimpleInputStreamReader::Internal{
public:
- class JStreamsBuffer: public jstreams::BufferedReader{
+ class JStreamsBuffer: public BufferedReaderImpl{
InputStream* input;
char utf8buf[6]; //< buffer used for converting utf8 characters
protected:
@@ -261,10 +327,10 @@
}else if ( ret == -1 )
return -1;
this->m_error = "Invalid multibyte sequence.";
- this->m_status = jstreams::Error;
+ this->m_status = CL_NS(util)::Error;
}else{
this->m_error = "Unexpected encoding";
- this->m_status = jstreams::Error;
+ this->m_status = CL_NS(util)::Error;
}
return -1;
}
@@ -276,7 +342,7 @@
for(i=0;i<space;i++){
c = readChar();
if ( c == -1 ){
- if ( this->m_status == jstreams::Ok ){
+ if ( this->m_status == CL_NS(util)::Ok ){
if ( i == 0 )
return -1;
break;
@@ -293,6 +359,7 @@
JStreamsBuffer(InputStream* input, int encoding){
this->input = input;
this->encoding = encoding;
+ setMinBufSize(1024);
}
~JStreamsBuffer(){
_CLDELETE(input);
@@ -315,6 +382,9 @@
SimpleInputStreamReader::SimpleInputStreamReader(){
internal = NULL;
}
+SimpleInputStreamReader::SimpleInputStreamReader(InputStream *i, int encoding){
+ internal = new Internal(i, encoding);
+}
void SimpleInputStreamReader::init(InputStream *i, int encoding){
internal = new Internal(i, encoding);
}
@@ -343,7 +413,7 @@
class FilteredBufferedReader::Internal{
public:
- class JStreamsFilteredBuffer: public jstreams::BufferedReader{
+ class JStreamsFilteredBuffer: public BufferedReaderImpl{
Reader* input;
bool deleteInput;
protected:
@@ -401,4 +471,67 @@
return internal->jsbuffer->_setMinBufSize(minbufsize);
}
+
+
+
+class FilteredBufferedInputStream::Internal{
+public:
+ class JStreamsFilteredBuffer: public BufferedInputStreamImpl{
+ InputStream* input;
+ bool deleteInput;
+ protected:
+ int32_t fillBuffer(signed char* start, int32_t space){
+ const signed char* buffer;
+ int32_t r = input->read(buffer, 1, space);
+ if ( r > 0 )
+ memcpy(start, buffer, r);
+ return r;
+ }
+ public:
+ JStreamsFilteredBuffer(InputStream* input, bool deleteInput){
+ this->input = input;
+ this->deleteInput = deleteInput;
+ }
+ ~JStreamsFilteredBuffer(){
+ if ( deleteInput )
+ _CLDELETE(input);
+ }
+ void _setMinBufSize(int32_t min){
+ this->setMinBufSize(min);
+ }
+ };
+ JStreamsFilteredBuffer* jsbuffer;
+
+ Internal(InputStream* input, bool deleteInput){
+ this->jsbuffer = new JStreamsFilteredBuffer(input, deleteInput);
+ }
+ ~Internal(){
+ delete jsbuffer;
+ }
+};
+FilteredBufferedInputStream::FilteredBufferedInputStream(InputStream* input, bool deleteInput){
+ internal = new Internal(input, deleteInput);
+}
+FilteredBufferedInputStream::~FilteredBufferedInputStream(){
+ delete internal;
+}
+int32_t FilteredBufferedInputStream::read(const signed char*& start, int32_t min, int32_t max){
+ return internal->jsbuffer->read(start,min,max);
+}
+int64_t FilteredBufferedInputStream::position(){
+ return internal->jsbuffer->position();
+}
+int64_t FilteredBufferedInputStream::reset(int64_t p){
+ return internal->jsbuffer->reset(p);
+}
+int64_t FilteredBufferedInputStream::skip(int64_t ntoskip){
+ return internal->jsbuffer->skip(ntoskip);
+}
+size_t FilteredBufferedInputStream::size(){
+ return internal->jsbuffer->size();
+}
+void FilteredBufferedInputStream::setMinBufSize(int32_t minbufsize){
+ return internal->jsbuffer->_setMinBufSize(minbufsize);
+}
+
CL_NS_END
Modified: branches/lucene2_3_2/src/core/CLucene/util/_bufferedstream.h
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/util/_bufferedstream.h 2009-03-30 20:54:55 UTC (rev 2972)
+++ branches/lucene2_3_2/src/core/CLucene/util/_bufferedstream.h 2009-03-30 20:57:49 UTC (rev 2973)
@@ -1,21 +1,21 @@
-#ifndef STRIGI_BUFFEREDSTREAM_H
-#define STRIGI_BUFFEREDSTREAM_H
+#ifndef JSTREAMS_BUFFEREDSTREAM_H
+#define JSTREAMS_BUFFEREDSTREAM_H
#include "_streambase.h"
#include "_streambuffer.h"
#include <cassert>
-namespace jstreams {
+CL_NS_DEF(util)
/**
- * @brief Abstract class providing a buffered input stream.
+ * @brief Abstract implementation class providing a buffered input stream.
*
* You can inherit this class to provide buffered access to a
* resource. You just need to implement fillBuffer, and
* BufferedStream will do the rest.
*/
template <class T>
-class BufferedStream : public StreamBase<T> {
+class BufferedStreamImpl : public StreamBase<T> {
private:
StreamBuffer<T> buffer;
bool finishedWritingToBuffer;
@@ -62,7 +62,7 @@
void setMinBufSize(int32_t s) {
buffer.makeSpace(s);
}
- BufferedStream<T>();
+ BufferedStreamImpl<T>();
public:
int32_t read(const T*& start, int32_t min, int32_t max);
int64_t reset(int64_t pos);
@@ -71,20 +71,20 @@
/** Abstract class for a buffered stream of bytes */
-typedef BufferedStream<signed char> BufferedInputStream;
+typedef BufferedStreamImpl<signed char> BufferedInputStreamImpl;
/** Abstract class for a buffered stream of Unicode characters */
-typedef BufferedStream<wchar_t> BufferedReader;
+typedef BufferedStreamImpl<wchar_t> BufferedReaderImpl;
template <class T>
-BufferedStream<T>::BufferedStream() {
+BufferedStreamImpl<T>::BufferedStreamImpl() {
finishedWritingToBuffer = false;
}
template <class T>
void
-BufferedStream<T>::writeToBuffer(int32_t ntoread, int32_t maxread) {
+BufferedStreamImpl<T>::writeToBuffer(int32_t ntoread, int32_t maxread) {
int32_t missing = ntoread - buffer.avail;
int32_t nwritten = 0;
while (missing > 0 && nwritten >= 0) {
@@ -107,7 +107,7 @@
}
template <class T>
int32_t
-BufferedStream<T>::read(const T*& start, int32_t min, int32_t max) {
+BufferedStreamImpl<T>::read(const T*& start, int32_t min, int32_t max) {
if (StreamBase<T>::m_status == Error) return -2;
if (StreamBase<T>::m_status == Eof) return -1;
@@ -143,7 +143,7 @@
}
template <class T>
int64_t
-BufferedStream<T>::reset(int64_t newpos) {
+BufferedStreamImpl<T>::reset(int64_t newpos) {
assert(newpos >= 0);
if (StreamBase<T>::m_status == Error) return -2;
// check to see if we have this position
@@ -158,7 +158,7 @@
}
template <class T>
int64_t
-BufferedStream<T>::skip(int64_t ntoskip) {
+BufferedStreamImpl<T>::skip(int64_t ntoskip) {
const T *begin;
int32_t nread;
int64_t skipped = 0;
@@ -174,6 +174,6 @@
return skipped;
}
-} // end namespace Strigi
+CL_NS_END
#endif
Modified: branches/lucene2_3_2/src/core/CLucene/util/_streambase.h
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/util/_streambase.h 2009-03-30 20:54:55 UTC (rev 2972)
+++ branches/lucene2_3_2/src/core/CLucene/util/_streambase.h 2009-03-30 20:57:49 UTC (rev 2973)
@@ -1,13 +1,13 @@
-#ifndef STRIGI_STREAMBASE_H
-#define STRIGI_STREAMBASE_H
+#ifndef JSTREAMS_STREAMBASE_H
+#define JSTREAMS_STREAMBASE_H
#include <stdio.h>
#include <string>
#define INT32MAX 0x7FFFFFFFL
-namespace jstreams {
+CL_NS_DEF(util)
/** Used to indicate the current status of a Stream */
enum StreamStatus {
@@ -173,13 +173,6 @@
};
-/** Abstract class for a stream of bytes */
-typedef StreamBase<char> InputStream;
-
-/** Abstract class for a stream of Unicode characters */
-typedef StreamBase<wchar_t> Reader;
-
-
template <class T>
int64_t
StreamBase<T>::skip(int64_t ntoskip) {
@@ -207,6 +200,6 @@
return skipped;
}
-} // end namespace Strigi
+CL_NS_END
#endif
Modified: branches/lucene2_3_2/src/core/CLucene/util/_streambuffer.h
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/util/_streambuffer.h 2009-03-30 20:54:55 UTC (rev 2972)
+++ branches/lucene2_3_2/src/core/CLucene/util/_streambuffer.h 2009-03-30 20:57:49 UTC (rev 2973)
@@ -1,11 +1,11 @@
-#ifndef STRIGI_STREAMBUFFER_H
-#define STRIGI_STREAMBUFFER_H
+#ifndef JSTREAM_STREAMBUFFER_H
+#define JSTREAM_STREAMBUFFER_H
#include <cstdlib>
#include <cstring>
-namespace jstreams {
+CL_NS_DEF(util)
/**
* @internal
@@ -157,6 +157,6 @@
return max;
}
-} // end namespace Strigi
+CL_NS_END
#endif
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-30 20:55:03
|
Revision: 2972
http://clucene.svn.sourceforge.net/clucene/?rev=2972&view=rev
Author: ustramooner
Date: 2009-03-30 20:54:55 +0000 (Mon, 30 Mar 2009)
Log Message:
-----------
check float<>byte conversions. check error handling properly
Modified Paths:
--------------
branches/lucene2_3_2/src/shared/CMakeLists.txt
branches/lucene2_3_2/src/shared/cmake/CheckErrorHandling.cmake
Added Paths:
-----------
branches/lucene2_3_2/src/shared/cmake/CheckFloatByte.cmake
branches/lucene2_3_2/src/shared/cmake/CheckFloatByte.cpp.in
Modified: branches/lucene2_3_2/src/shared/CMakeLists.txt
===================================================================
--- branches/lucene2_3_2/src/shared/CMakeLists.txt 2009-03-30 20:51:34 UTC (rev 2971)
+++ branches/lucene2_3_2/src/shared/CMakeLists.txt 2009-03-30 20:54:55 UTC (rev 2972)
@@ -28,6 +28,7 @@
INCLUDE (DefineStaticSyntax)
INCLUDE (DefineMAXPATHValue)
+INCLUDE (CheckFloatByte)
INCLUDE (CheckErrorHandling)
INCLUDE (CheckHashmaps)
INCLUDE (CheckNamespace)
@@ -86,9 +87,8 @@
wcscmp wcsncmp wcscspn )
#todo: wcstoq is bsd equiv of wcstoll, we can use that...
-CHECK_OPTIONAL_FUNCTIONS( wcsupr wcscasecmp wcsicmp wcstoll wprintf lltow wcstod strupr strlwr lltoa strtoll gettimeofday _vsnwprintf
- "MapViewOfFile(0,0,0,0,0)"
- mmap
+CHECK_OPTIONAL_FUNCTIONS( wcsupr wcscasecmp wcsicmp wcstoll wprintf lltow
+ wcstod strupr strlwr lltoa strtoll gettimeofday _vsnwprintf mmap "MapViewOfFile(0,0,0,0,0)"
)
#make decisions about which functions to use...
@@ -220,6 +220,9 @@
#Check that we can handle try/catch
CHECK_HAVE_FUNCTION_TRY_BLOCKS (_CL_HAVE_TRY_BLOCKS)
+#check that we support new float byte<->float conversions
+CHECK_FLOAT_BYTE_WORKS(_CL_HAVE_NO_FLOAT_BYTE, 1)
+
#check how to use hashmaps
CHECK_HASH_MAPS (CL_NS_HASHING_VALUE LUCENE_DISABLE_HASHING)
Modified: branches/lucene2_3_2/src/shared/cmake/CheckErrorHandling.cmake
===================================================================
--- branches/lucene2_3_2/src/shared/cmake/CheckErrorHandling.cmake 2009-03-30 20:51:34 UTC (rev 2971)
+++ branches/lucene2_3_2/src/shared/cmake/CheckErrorHandling.cmake 2009-03-30 20:54:55 UTC (rev 2972)
@@ -6,7 +6,7 @@
CHECK_CXX_SOURCE_RUNS("
void foo() { try{ return; } catch( ... ){} }
int main(){ foo(); return 0; }" ${result})
- IF ( NOT _CL_HAVE_TRY_BLOCKS )
- SET ( ${result} 1 )
- ENDIF ( NOT _CL_HAVE_TRY_BLOCKS )
+ IF ( NOT ${result} )
+ SET ( ${result} 1 FORCE)
+ ENDIF ( NOT ${result} )
ENDMACRO ( CHECK_HAVE_FUNCTION_TRY_BLOCKS )
Added: branches/lucene2_3_2/src/shared/cmake/CheckFloatByte.cmake
===================================================================
--- branches/lucene2_3_2/src/shared/cmake/CheckFloatByte.cmake (rev 0)
+++ branches/lucene2_3_2/src/shared/cmake/CheckFloatByte.cmake 2009-03-30 20:54:55 UTC (rev 2972)
@@ -0,0 +1,37 @@
+# - Check if our methods for converting from floats to bytes and back work.
+# CHECK_FLOAT_BYTE_WORKS(RESULT reverse)
+# reverse: set to false if the check succeeds
+#
+# CMAKE_REQUIRED_FLAGS = string of compile command line flags
+# CMAKE_REQUIRED_DEFINITIONS = list of macros to define (-DFOO=bar)
+# CMAKE_REQUIRED_INCLUDES = list of include directories
+# CMAKE_REQUIRED_LIBRARIES = list of libraries to link
+# CMAKE_EXTRA_INCLUDE_FILES = list of extra includes to check in
+
+MACRO(CHECK_FLOAT_BYTE_WORKS RESULT reverse)
+ IF("${RESULT}" MATCHES "^${RESULT}$")
+ MESSAGE(STATUS "Checking support new float byte<->float conversions")
+
+ CONFIGURE_FILE("${clucene-shared_SOURCE_DIR}/cmake/CheckFloatByte.cpp.in"
+ "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/CheckFloatByte.cpp" IMMEDIATE @ONLY)
+
+ TRY_COMPILE(${RESULT}
+ ${CMAKE_BINARY_DIR}
+ "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/CheckFloatByte.cpp"
+ OUTPUT_VARIABLE OUTPUT)
+ IF(${RESULT})
+ MESSAGE(STATUS "Checking support new float byte<->float conversions - yes")
+ ELSE(${RESULT})
+ MESSAGE(STATUS "Checking support new float byte<->float conversions - no")
+ ENDIF(${RESULT})
+
+ #reverse decision if required.
+ IF (${reverse})
+ IF (${RESULT})
+ SET ( ${RESULT} 0 )
+ ELSE (${RESULT})
+ SET ( ${RESULT} 1 )
+ ENDIF (${RESULT})
+ ENDIF (${reverse})
+ ENDIF("${RESULT}" MATCHES "^${RESULT}$")
+ENDMACRO(CHECK_FLOAT_BYTE_WORKS)
Added: branches/lucene2_3_2/src/shared/cmake/CheckFloatByte.cpp.in
===================================================================
--- branches/lucene2_3_2/src/shared/cmake/CheckFloatByte.cpp.in (rev 0)
+++ branches/lucene2_3_2/src/shared/cmake/CheckFloatByte.cpp.in 2009-03-30 20:54:55 UTC (rev 2972)
@@ -0,0 +1,81 @@
+ typedef long double float_t;
+ typedef long int32_t;
+ typedef char uint8_t;
+
+ //float to bits conversion utilities...
+ union clvalue {
+ int32_t i;
+ float f; //must use a float type, else types dont match up
+ };
+
+ int32_t floatToIntBits(float_t value)
+ {
+ clvalue u;
+ int32_t e, f;
+ u.f = value;
+ e = u.i & 0x7f800000;
+ f = u.i & 0x007fffff;
+
+ if (e == 0x7f800000 && f != 0)
+ u.i = 0x7fc00000;
+
+ return u.i;
+ }
+ float_t intBitsToFloat(int32_t bits)
+ {
+ clvalue u;
+ u.i = bits;
+ return u.f;
+ }
+
+ float_t byteToFloat(uint8_t b) {
+ if (b == 0) // zero is a special case
+ return 0.0f;
+ int32_t mantissa = b & 7;
+ int32_t exponent = (b >> 3) & 31;
+ int32_t bits = ((exponent+(63-15)) << 24) | (mantissa << 21);
+ return intBitsToFloat(bits);
+ }
+
+ uint8_t floatToByte(float_t f) {
+ if (f < 0.0f) // round negatives up to zero
+ f = 0.0f;
+
+ if (f == 0.0f) // zero is a special case
+ return 0;
+
+ int32_t bits = floatToIntBits(f); // parse float_t into parts
+ int32_t mantissa = (bits & 0xffffff) >> 21;
+ int32_t exponent = (((bits >> 24) & 0x7f) - 63) + 15;
+
+ if (exponent > 31) { // overflow: use max value
+ exponent = 31;
+ mantissa = 7;
+ }
+
+ if (exponent < 0) { // underflow: use min value
+ exponent = 0;
+ mantissa = 1;
+ }
+
+ return (uint8_t)((exponent << 3) | mantissa); // pack into a uint8_t
+ }
+
+
+#ifdef __CLASSIC_C__
+int main(){
+ int ac;
+ char*av[];
+#else
+int main(int ac, char*av[]){
+#endif
+ //well known conversion
+ if ( floatToByte(0.5f) != 120 )
+ return 1;
+
+ //converting back works?
+ if ( floatToByte(byteToFloat(57)) != 57 )
+ return 1;
+
+ return 0;
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-30 20:51:51
|
Revision: 2971
http://clucene.svn.sourceforge.net/clucene/?rev=2971&view=rev
Author: ustramooner
Date: 2009-03-30 20:51:34 +0000 (Mon, 30 Mar 2009)
Log Message:
-----------
remove _CL_HAVE_NO_FUNCTION_TRY_BLOCKS - not necessary
Modified Paths:
--------------
branches/lucene2_3_2/src/shared/CLucene/_clucene-config.h.cmake
Modified: branches/lucene2_3_2/src/shared/CLucene/_clucene-config.h.cmake
===================================================================
--- branches/lucene2_3_2/src/shared/CLucene/_clucene-config.h.cmake 2009-03-30 20:50:22 UTC (rev 2970)
+++ branches/lucene2_3_2/src/shared/CLucene/_clucene-config.h.cmake 2009-03-30 20:51:34 UTC (rev 2971)
@@ -76,13 +76,8 @@
/* CMake will determine these specifics. Things like bugs, etc */
/* Does not support new float byte<->float conversions */
-//todo: not being checked for...
-#cmakedefine _CL_HAVE_NO_FLOAT_BYTE
+#cmakedefine _CL_HAVE_NO_FLOAT_BYTE 1
-/* Does not support try/catch blocks */
-//todo: this is not checked properly
-//#cmakedefine _CL_HAVE_NO_FUNCTION_TRY_BLOCKS
-
/* Define if recursive pthread mutexes are available */
#cmakedefine _CL_HAVE_PTHREAD_MUTEX_RECURSIVE 1
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-30 20:50:32
|
Revision: 2970
http://clucene.svn.sourceforge.net/clucene/?rev=2970&view=rev
Author: ustramooner
Date: 2009-03-30 20:50:22 +0000 (Mon, 30 Mar 2009)
Log Message:
-----------
cleanup code
Modified Paths:
--------------
branches/lucene2_3_2/src/core/CLucene/util/ThreadLocal.cpp
Modified: branches/lucene2_3_2/src/core/CLucene/util/ThreadLocal.cpp
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/util/ThreadLocal.cpp 2009-03-30 20:46:36 UTC (rev 2969)
+++ branches/lucene2_3_2/src/core/CLucene/util/ThreadLocal.cpp 2009-03-30 20:50:22 UTC (rev 2970)
@@ -25,20 +25,13 @@
#if defined(_CL_HAVE_WIN32_THREADS)
#define INIT_THREAD(ret) ret=true
extern "C"{
- #ifndef _WINBASE_
- /*__declspec(dllimport) _cl_dword_t __stdcall TlsAlloc( );
- __declspec(dllimport) void* __stdcall TlsGetValue(_cl_dword_t dwTlsIndex );
- __declspec(dllimport) bool __stdcall TlsSetValue( _cl_dword_t dwTlsIndex, void* lpTlsValue );
- __declspec(dllimport) bool __stdcall TlsFree( _cl_dword_t dwTlsIndex );*/
- #define DLL_THREAD_DETACH 3
- #endif //_WINBASE_
//todo: move this to StdHeader and make it usable by other functions...
bool __stdcall DllMain( unsigned short hinstDLL, // DLL module handle
_cl_dword_t fdwReason, // reason called
void*) // reserved
{
- if ( fdwReason == DLL_THREAD_DETACH )
+ if ( fdwReason == 3 )
_ThreadLocal::UnregisterCurrentThread();
return true;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-30 20:46:55
|
Revision: 2969
http://clucene.svn.sourceforge.net/clucene/?rev=2969&view=rev
Author: ustramooner
Date: 2009-03-30 20:46:36 +0000 (Mon, 30 Mar 2009)
Log Message:
-----------
use NULL instead
Modified Paths:
--------------
branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/LanguageBasedAnalyzer.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/LanguageBasedAnalyzer.h
Modified: branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/LanguageBasedAnalyzer.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/LanguageBasedAnalyzer.cpp 2009-03-29 13:47:59 UTC (rev 2968)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/LanguageBasedAnalyzer.cpp 2009-03-30 20:46:36 UTC (rev 2969)
@@ -15,7 +15,10 @@
LanguageBasedAnalyzer::LanguageBasedAnalyzer(const TCHAR* language, bool stem)
{
- _tcsncpy(lang,language,100);
+ if ( language == NULL )
+ _tcsncpy(lang,LUCENE_BLANK_STRING,100);
+ else
+ _tcsncpy(lang,language,100);
this->stem = stem;
}
LanguageBasedAnalyzer::~LanguageBasedAnalyzer(){
Modified: branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/LanguageBasedAnalyzer.h
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/LanguageBasedAnalyzer.h 2009-03-29 13:47:59 UTC (rev 2968)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/LanguageBasedAnalyzer.h 2009-03-30 20:46:36 UTC (rev 2969)
@@ -24,7 +24,7 @@
TCHAR lang[100];
bool stem;
public:
- LanguageBasedAnalyzer(const TCHAR* language=LUCENE_BLANK_STRING, bool stem=true);
+ LanguageBasedAnalyzer(const TCHAR* language=NULL, bool stem=true);
~LanguageBasedAnalyzer();
void setLanguage(const TCHAR* language);
void setStem(bool stem);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-29 13:48:01
|
Revision: 2968
http://clucene.svn.sourceforge.net/clucene/?rev=2968&view=rev
Author: ustramooner
Date: 2009-03-29 13:47:59 +0000 (Sun, 29 Mar 2009)
Log Message:
-----------
fixes for problems discovered by the dist-test.sh script
Modified Paths:
--------------
branches/lucene2_3_2/dist-test.sh
branches/lucene2_3_2/src/core/CLucene/search/FieldSortedHitQueue.h
branches/lucene2_3_2/src/core/CLucene/store/FSDirectory.cpp
branches/lucene2_3_2/src/core/CLucene/store/LockFactory.h
branches/lucene2_3_2/src/shared/CLucene/config/repl_wchar.h
branches/lucene2_3_2/src/test/test.h
Modified: branches/lucene2_3_2/dist-test.sh
===================================================================
--- branches/lucene2_3_2/dist-test.sh 2009-03-29 10:57:30 UTC (rev 2967)
+++ branches/lucene2_3_2/dist-test.sh 2009-03-29 13:47:59 UTC (rev 2968)
@@ -71,7 +71,7 @@
#check to see that no #ifdefs exist in headers that don't belong
function checkForIfdefs {
I=0
- grep "#if" $1| while read line; do
+ grep "#if" $1| grep -v "_UCS2" |grep -v "_CL_HAVE_" |grep -v "_ASCII" |grep -v "_WIN32" |grep -v "_WIN64" | while read line; do
I=`expr $I + 1`
if [ $I -gt 1 ]; then
echo $1 has invalid ifdef: $line
@@ -102,7 +102,9 @@
ln -s "`cd "$DN" && pwd`/$BH" "$TMP/${H:3}" 2>/dev/null
#create pub-headers.cpp
- echo "#include \"${H:7}\"" >>$TMP/pub-headers.cpp
+ if [ "${H:7}" != "core/CLucene/util/Reader.h" ]; then
+ echo "#include \"${H:7}\"" >>$TMP/pub-headers.cpp
+ fi
fi
done
fi
@@ -147,11 +149,12 @@
fi
#test that each header compiles independently...
- if [ $t_c_h == 1 ]; then
+ if [ $t_c_h == 1 ] && [ "${H:7}" != "disttest/src/core/CLucene/util/Reader.h" ]; then
+ echo "Test that $H compiles seperately..."
echo "#include \"CLucene/StdHeader.h"\" >$TMP/pub-header.cpp
echo "#include \"$H"\" >>$TMP/pub-header.cpp
echo "int main(){return 0;}" >>$TMP/pub-header.cpp
- g++ -I. -I$TMP/src/shared -I$TMP/src/core $TMP/pub-header.cpp
+ g++ -I. -I$TMP/src/shared -I./src/shared -I$TMP/src/core $TMP/pub-header.cpp
if [ $? != 0 ]; then
FAIL=1;
fi
Modified: branches/lucene2_3_2/src/core/CLucene/search/FieldSortedHitQueue.h
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/search/FieldSortedHitQueue.h 2009-03-29 10:57:30 UTC (rev 2967)
+++ branches/lucene2_3_2/src/core/CLucene/search/FieldSortedHitQueue.h 2009-03-29 13:47:59 UTC (rev 2968)
@@ -11,14 +11,10 @@
CL_CLASS_DEF(search,FieldDoc)
CL_CLASS_DEF(search,SortComparatorSource)
CL_CLASS_DEF(search,SortField)
-//#include "FieldCache.h"
-//#include "Sort.h"
-//#include "FieldDocSortedHitQueue.h"
#include "FieldDoc.h" //required to expose destructor
-//#include "SearchHeader.h"
-//#include "FieldCacheImpl.h"
#include "CLucene/util/PriorityQueue.h"
#include "CLucene/util/Equators.h"
+#include "CLucene/LuceneThreads.h"
CL_CLASS_DEF(index,IndexReader)
Modified: branches/lucene2_3_2/src/core/CLucene/store/FSDirectory.cpp
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/store/FSDirectory.cpp 2009-03-29 10:57:30 UTC (rev 2967)
+++ branches/lucene2_3_2/src/core/CLucene/store/FSDirectory.cpp 2009-03-29 13:47:59 UTC (rev 2968)
@@ -577,7 +577,7 @@
#endif
}
- bool FSDirectory::openInput(const char * name, lucene::store::IndexInput *& ret, CLuceneError& error, int32_t bufferSize)
+ bool FSDirectory::openInput(const char * name, IndexInput *& ret, CLuceneError& error, int32_t bufferSize)
{
CND_PRECONDITION(directory[0]!=0,"directory is not open")
char fl[CL_MAX_DIR];
Modified: branches/lucene2_3_2/src/core/CLucene/store/LockFactory.h
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/store/LockFactory.h 2009-03-29 10:57:30 UTC (rev 2967)
+++ branches/lucene2_3_2/src/core/CLucene/store/LockFactory.h 2009-03-29 13:47:59 UTC (rev 2968)
@@ -7,9 +7,8 @@
#ifndef _lucene_store_LockFactory_
#define _lucene_store_LockFactory_
-//#include "Lock.h"
+#include "CLucene/LuceneThreads.h"
-
CL_CLASS_DEF(store,LuceneLock)
CL_CLASS_DEF(store,NoLock)
Modified: branches/lucene2_3_2/src/shared/CLucene/config/repl_wchar.h
===================================================================
--- branches/lucene2_3_2/src/shared/CLucene/config/repl_wchar.h 2009-03-29 10:57:30 UTC (rev 2967)
+++ branches/lucene2_3_2/src/shared/CLucene/config/repl_wchar.h 2009-03-29 13:47:59 UTC (rev 2968)
@@ -8,6 +8,7 @@
#define _lucene_repl_wchar_h
#include <stdarg.h>
+#include "repl_tchar.h"
#ifdef _CL_HAVE_STRING_H
#include <string.h>
#endif
Modified: branches/lucene2_3_2/src/test/test.h
===================================================================
--- branches/lucene2_3_2/src/test/test.h 2009-03-29 10:57:30 UTC (rev 2967)
+++ branches/lucene2_3_2/src/test/test.h 2009-03-29 13:47:59 UTC (rev 2968)
@@ -18,6 +18,7 @@
#include "CLucene/store/Lock.h"
#include "CLucene/index/TermVector.h"
#include "CLucene/queryParser/MultiFieldQueryParser.h"
+#include <string.h>
#define LUCENE_INT64_MAX_SHOULDBE _ILONGLONG(0x7FFFFFFFFFFFFFFF)
#define LUCENE_INT64_MIN_SHOULDBE (-LUCENE_INT64_MAX_SHOULDBE - _ILONGLONG(1) )
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-29 10:57:34
|
Revision: 2967
http://clucene.svn.sourceforge.net/clucene/?rev=2967&view=rev
Author: ustramooner
Date: 2009-03-29 10:57:30 +0000 (Sun, 29 Mar 2009)
Log Message:
-----------
fixed signed/unsigned warning
Modified Paths:
--------------
branches/lucene2_3_2/src/core/CLucene/search/PrefixQuery.cpp
Modified: branches/lucene2_3_2/src/core/CLucene/search/PrefixQuery.cpp
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/search/PrefixQuery.cpp 2009-03-29 10:56:58 UTC (rev 2966)
+++ branches/lucene2_3_2/src/core/CLucene/search/PrefixQuery.cpp 2009-03-29 10:57:30 UTC (rev 2967)
@@ -94,7 +94,7 @@
const TCHAR* prefixField = prefix->field();
const TCHAR* tmp;
size_t i;
- int32_t prefixLen = prefix->textLength();
+ size_t prefixLen = prefix->textLength();
do {
lastTerm = enumerator->term();
if (lastTerm != NULL &&
@@ -235,7 +235,7 @@
const TCHAR* prefixField = prefix->field();
const TCHAR* tmp;
size_t i;
- int32_t prefixLen = prefix->textLength();
+ size_t prefixLen = prefix->textLength();
Term* lastTerm = NULL;
try{
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-29 10:57:02
|
Revision: 2966
http://clucene.svn.sourceforge.net/clucene/?rev=2966&view=rev
Author: ustramooner
Date: 2009-03-29 10:56:58 +0000 (Sun, 29 Mar 2009)
Log Message:
-----------
fixed compile errors relating to new branch code
Modified Paths:
--------------
branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/LanguageBasedAnalyzer.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/cjk/CJKAnalyzer.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/highlighter/Highlighter.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/highlighter/TokenSources.cpp
branches/lucene2_3_2/src/contribs-lib/CLucene/snowball/Snowball.cpp
Modified: branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/LanguageBasedAnalyzer.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/LanguageBasedAnalyzer.cpp 2009-03-29 10:56:10 UTC (rev 2965)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/LanguageBasedAnalyzer.cpp 2009-03-29 10:56:58 UTC (rev 2966)
@@ -31,7 +31,12 @@
if ( _tcscmp(lang, _T("cjk"))==0 ){
ret = _CLNEW CL_NS2(analysis,cjk)::CJKTokenizer(reader);
}else{
- ret = _CLNEW StandardTokenizer(reader);
+ BufferedReader* bufferedReader = reader->__asBufferedReader();
+ if ( bufferedReader == NULL )
+ ret = _CLNEW StandardTokenizer( _CLNEW FilteredBufferedReader(reader, false), true );
+ else
+ ret = _CLNEW StandardTokenizer(bufferedReader);
+
ret = _CLNEW StandardFilter(ret,true);
if ( stem )
Modified: branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/cjk/CJKAnalyzer.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/cjk/CJKAnalyzer.cpp 2009-03-29 10:56:10 UTC (rev 2965)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/analysis/cjk/CJKAnalyzer.cpp 2009-03-29 10:56:58 UTC (rev 2966)
@@ -1,6 +1,6 @@
#include "CLucene/_ApiHeader.h"
#include "CJKAnalyzer.h"
-#include "CLucene/util/Reader.h"
+#include "CLucene/util/CLStreams.h"
CL_NS_DEF2(analysis,cjk)
CL_NS_USE(analysis)
@@ -36,7 +36,7 @@
offset++;
if (bufferIndex >= dataLen) {
- dataLen = input->read(ioBuffer);
+ dataLen = input->read(ioBuffer, 1, LUCENE_IO_BUFFER_SIZE);
bufferIndex = 0;
}
Modified: branches/lucene2_3_2/src/contribs-lib/CLucene/highlighter/Highlighter.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/highlighter/Highlighter.cpp 2009-03-29 10:56:10 UTC (rev 2965)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/highlighter/Highlighter.cpp 2009-03-29 10:56:58 UTC (rev 2966)
@@ -28,7 +28,7 @@
#include "CLucene/analysis/AnalysisHeader.h"
#include "CLucene/util/PriorityQueue.h"
#include "CLucene/util/StringBuffer.h"
-#include "CLucene/util/Reader.h"
+#include "CLucene/util/CLStreams.h"
CL_NS_DEF2(search,highlight)
CL_NS_USE(analysis)
Modified: branches/lucene2_3_2/src/contribs-lib/CLucene/highlighter/TokenSources.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/highlighter/TokenSources.cpp 2009-03-29 10:56:10 UTC (rev 2965)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/highlighter/TokenSources.cpp 2009-03-29 10:56:58 UTC (rev 2966)
@@ -17,8 +17,8 @@
#include "CLucene/_ApiHeader.h"
#include "TokenSources.h"
-#include "CLucene/util/VoidList.h"
-#include "CLucene/util/Reader.h"
+#include "CLucene/util/VoidList.h"
+#include "CLucene/util/CLStreams.h"
#include "CLucene/index/IndexReader.h"
#include "CLucene/index/TermVector.h"
#include "CLucene/document/Document.h"
Modified: branches/lucene2_3_2/src/contribs-lib/CLucene/snowball/Snowball.cpp
===================================================================
--- branches/lucene2_3_2/src/contribs-lib/CLucene/snowball/Snowball.cpp 2009-03-29 10:56:10 UTC (rev 2965)
+++ branches/lucene2_3_2/src/contribs-lib/CLucene/snowball/Snowball.cpp 2009-03-29 10:56:58 UTC (rev 2966)
@@ -7,6 +7,7 @@
#include "CLucene/analysis/standard/StandardFilter.h"
CL_NS_USE(analysis)
+CL_NS_USE(util)
CL_NS_USE2(analysis,standard)
CL_NS_DEF2(analysis,snowball)
@@ -35,8 +36,14 @@
/** Constructs a {@link StandardTokenizer} filtered by a {@link
StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}. */
TokenStream* SnowballAnalyzer::tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader) {
- TokenStream* result = _CLNEW StandardTokenizer(reader);
- result = _CLNEW StandardFilter(result, true);
+ TokenStream* result = NULL;
+ BufferedReader* bufferedReader = reader->__asBufferedReader();
+ if ( bufferedReader == NULL )
+ result = _CLNEW StandardTokenizer( _CLNEW FilteredBufferedReader(reader, false), true );
+ else
+ result = _CLNEW StandardTokenizer(bufferedReader);
+
+ result = _CLNEW StandardFilter(result, true);
result = _CLNEW CL_NS(analysis)::LowerCaseFilter(result, true);
if (stopSet != NULL)
result = _CLNEW CL_NS(analysis)::StopFilter(result, true, stopSet);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-29 10:56:15
|
Revision: 2965
http://clucene.svn.sourceforge.net/clucene/?rev=2965&view=rev
Author: ustramooner
Date: 2009-03-29 10:56:10 +0000 (Sun, 29 Mar 2009)
Log Message:
-----------
added NumberTools
Modified Paths:
--------------
branches/lucene2_3_2/src/core/CLucene/CLMonolithic.cpp
branches/lucene2_3_2/src/test/CLMonolithic_Test.cpp
Modified: branches/lucene2_3_2/src/core/CLucene/CLMonolithic.cpp
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/CLMonolithic.cpp 2009-03-29 10:55:56 UTC (rev 2964)
+++ branches/lucene2_3_2/src/core/CLucene/CLMonolithic.cpp 2009-03-29 10:56:10 UTC (rev 2965)
@@ -20,6 +20,7 @@
#include "CLucene/document/DateField.cpp"
#include "CLucene/document/DateTools.cpp"
#include "CLucene/document/Document.cpp"
+#include "CLucene/document/NumberTools.cpp"
#include "CLucene/document/Field.cpp"
#include "CLucene/index/CompoundFile.cpp"
#include "CLucene/index/DocumentWriter.cpp"
Modified: branches/lucene2_3_2/src/test/CLMonolithic_Test.cpp
===================================================================
--- branches/lucene2_3_2/src/test/CLMonolithic_Test.cpp 2009-03-29 10:55:56 UTC (rev 2964)
+++ branches/lucene2_3_2/src/test/CLMonolithic_Test.cpp 2009-03-29 10:56:10 UTC (rev 2965)
@@ -24,6 +24,7 @@
#include "util/English.cpp"
#include "debug/TestError.cpp"
#include "document/TestDocument.cpp"
+#include "document/TestNumberTools.cpp"
#include "queryParser/TestQueryParser.cpp"
#include "queryParser/TestMultiFieldQueryParser.cpp"
#include "search/TestSearch.cpp"
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-29 10:56:00
|
Revision: 2964
http://clucene.svn.sourceforge.net/clucene/?rev=2964&view=rev
Author: ustramooner
Date: 2009-03-29 10:55:56 +0000 (Sun, 29 Mar 2009)
Log Message:
-----------
fix comment
Modified Paths:
--------------
branches/lucene2_3_2/src/core/CLucene/util/Reader.h
Modified: branches/lucene2_3_2/src/core/CLucene/util/Reader.h
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/util/Reader.h 2009-03-29 10:37:56 UTC (rev 2963)
+++ branches/lucene2_3_2/src/core/CLucene/util/Reader.h 2009-03-29 10:55:56 UTC (rev 2964)
@@ -11,7 +11,7 @@
CL_NS_DEF(util)
#error Reader has been refactored. It is recommended that you use strigi streams
-#error for all input into CLucene. If, however, you don't want to use that dependency,
+#error for all input into CLucene. If, however, you dont want to use that dependency,
#error then you'll have to refactor your current code. The jstreams namespace
#error was completely removed
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-29 10:38:01
|
Revision: 2963
http://clucene.svn.sourceforge.net/clucene/?rev=2963&view=rev
Author: ustramooner
Date: 2009-03-29 10:37:56 +0000 (Sun, 29 Mar 2009)
Log Message:
-----------
fix bad commit
Modified Paths:
--------------
branches/lucene2_3_2/src/test/search/TestTermVector.cpp
Modified: branches/lucene2_3_2/src/test/search/TestTermVector.cpp
===================================================================
--- branches/lucene2_3_2/src/test/search/TestTermVector.cpp 2009-03-29 10:36:00 UTC (rev 2962)
+++ branches/lucene2_3_2/src/test/search/TestTermVector.cpp 2009-03-29 10:37:56 UTC (rev 2963)
@@ -187,7 +187,7 @@
//float_t coord = sim.coord()
//System.out.println("TF: " + tf + " IDF: " + idf + " LenNorm: " + lNorm);
const TCHAR** vTerms = vector->getTerms();
- const ValueArray<int32_t>* freqs = vector->getTermFrequencies();
+ const Array<int32_t>* freqs = vector->getTermFrequencies();
int32_t i=0;
while ( vTerms && vTerms[i] != NULL )
{
@@ -225,7 +225,7 @@
CLUCENE_ASSERT(vector != NULL);
//_tprintf(_T("Vector: %s\n"),vector);
const TCHAR** terms = vector->getTerms();
- const ValueArray<int32_t>* freqs = vector->getTermFrequencies();
+ const Array<int32_t>* freqs = vector->getTermFrequencies();
CLUCENE_ASSERT(terms != NULL);
int termsLength = 0;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-29 10:36:06
|
Revision: 2962
http://clucene.svn.sourceforge.net/clucene/?rev=2962&view=rev
Author: ustramooner
Date: 2009-03-29 10:36:00 +0000 (Sun, 29 Mar 2009)
Log Message:
-----------
various demo code cleanup
Modified Paths:
--------------
branches/lucene2_3_2/src/demo/IndexFiles.cpp
branches/lucene2_3_2/src/demo/Main.cpp
branches/lucene2_3_2/src/demo/SearchFiles.cpp
Modified: branches/lucene2_3_2/src/demo/IndexFiles.cpp
===================================================================
--- branches/lucene2_3_2/src/demo/IndexFiles.cpp 2009-03-29 10:12:47 UTC (rev 2961)
+++ branches/lucene2_3_2/src/demo/IndexFiles.cpp 2009-03-29 10:36:00 UTC (rev 2962)
@@ -15,8 +15,7 @@
#include <fstream>
#include <sys/stat.h>
#include <cctype>
-#include <string>
-
+#include <string.h>
using namespace std;
using namespace lucene::index;
using namespace lucene::analysis;
@@ -137,5 +136,5 @@
writer->close();
_CLDELETE(writer);
- printf("Indexing took: %d ms.\n\n", Misc::currentTimeMillis() - str);
+ printf("Indexing took: %d ms.\n\n", (int32_t)(Misc::currentTimeMillis() - str));
}
Modified: branches/lucene2_3_2/src/demo/Main.cpp
===================================================================
--- branches/lucene2_3_2/src/demo/Main.cpp 2009-03-29 10:12:47 UTC (rev 2961)
+++ branches/lucene2_3_2/src/demo/Main.cpp 2009-03-29 10:36:00 UTC (rev 2962)
@@ -18,6 +18,7 @@
#endif
#include <iostream>
+#include <string.h>
using namespace std;
using namespace lucene::util;
@@ -40,12 +41,14 @@
printf("Location of text files to be indexed: ");
char files[250];
- fgets(files,250,stdin);
+ char* tmp = fgets(files,250,stdin);
+ if ( tmp == NULL ) return 1;
files[strlen(files)-1] = 0;
printf("Location to store the clucene index: ");
char ndx[250];
- fgets(ndx,250,stdin);
+ tmp = fgets(ndx,250,stdin);
+ if ( tmp == NULL ) return 1;
ndx[strlen(ndx)-1] = 0;
IndexFiles(files,ndx,true);
@@ -54,9 +57,9 @@
DeleteFiles(ndx);
}catch(CLuceneError& err){
- printf(err.what());
+ printf("Error: %s\n", err.what());
}catch(...){
- printf("Unknown error");
+ printf("Unknown error\n");
}
_lucene_shutdown(); //clears all static memory
@@ -68,6 +71,6 @@
// _crtBreakAlloc
//for linux, use valgrind
- printf ("\n\nTime taken: %d\n\n",Misc::currentTimeMillis() - str);
+ printf ("\n\nTime taken: %d\n\n", (int32_t)(Misc::currentTimeMillis() - str));
return 0;
}
Modified: branches/lucene2_3_2/src/demo/SearchFiles.cpp
===================================================================
--- branches/lucene2_3_2/src/demo/SearchFiles.cpp 2009-03-29 10:12:47 UTC (rev 2961)
+++ branches/lucene2_3_2/src/demo/SearchFiles.cpp 2009-03-29 10:36:00 UTC (rev 2962)
@@ -31,7 +31,8 @@
IndexSearcher s(index);
while (true) {
printf("Enter query string: ");
- fgets(line,80,stdin);
+ char* tmp = fgets(line,80,stdin);
+ if ( tmp == NULL ) continue;
line[strlen(line)-1]=0;
if ( strlen(line) == 0 )
@@ -45,7 +46,7 @@
uint64_t str = Misc::currentTimeMillis();
Hits* h = s.search(q);
- uint64_t srch = Misc::currentTimeMillis() - str;
+ uint32_t srch = (int32_t)(Misc::currentTimeMillis() - str);
str = Misc::currentTimeMillis();
for ( int32_t i=0;i<h->length();i++ ){
@@ -56,7 +57,7 @@
}
printf("\n\nSearch took: %d ms.\n", srch);
- printf("Screen dump took: %d ms.\n\n", Misc::currentTimeMillis() - str);
+ printf("Screen dump took: %d ms.\n\n", (int32_t)(Misc::currentTimeMillis() - str));
_CLDELETE(h);
_CLDELETE(q);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-29 10:12:51
|
Revision: 2961
http://clucene.svn.sourceforge.net/clucene/?rev=2961&view=rev
Author: ustramooner
Date: 2009-03-29 10:12:47 +0000 (Sun, 29 Mar 2009)
Log Message:
-----------
remove confusion about how cmake works (for those used to the configure make make install dance)
Added Paths:
-----------
branches/lucene2_3_2/configure
Added: branches/lucene2_3_2/configure
===================================================================
--- branches/lucene2_3_2/configure (rev 0)
+++ branches/lucene2_3_2/configure 2009-03-29 10:12:47 UTC (rev 2961)
@@ -0,0 +1,2 @@
+cmake .
+
Property changes on: branches/lucene2_3_2/configure
___________________________________________________________________
Added: svn:executable
+ *
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-29 09:26:10
|
Revision: 2960
http://clucene.svn.sourceforge.net/clucene/?rev=2960&view=rev
Author: ustramooner
Date: 2009-03-29 09:26:05 +0000 (Sun, 29 Mar 2009)
Log Message:
-----------
fix realpath usage when directory doesn't exist
Modified Paths:
--------------
branches/lucene2_3_2/src/core/CLucene/store/FSDirectory.cpp
Modified: branches/lucene2_3_2/src/core/CLucene/store/FSDirectory.cpp
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/store/FSDirectory.cpp 2009-03-29 08:54:12 UTC (rev 2959)
+++ branches/lucene2_3_2/src/core/CLucene/store/FSDirectory.cpp 2009-03-29 09:26:05 UTC (rev 2960)
@@ -353,10 +353,12 @@
refCount(0),
useMMap(LUCENE_USE_MMAP)
{
- directory = _realpath(path,directory);//set a realpath so that if we change directory, we can still function
- if ( !directory || !*directory ){
- strcpy(directory,path);
- }
+ char* tmpdirectory = _realpath(path,directory);//set a realpath so that if we change directory, we can still function
+ if ( !tmpdirectory || !*tmpdirectory ){
+ strncpy(directory,path, CL_MAX_PATH);
+ }else{
+ directory = tmpdirectory; //repoint tmpdirectory at directory
+ }
bool doClearLockID = false;
@@ -380,10 +382,10 @@
}
if (!Misc::dir_Exists(directory)){
- char* err = _CL_NEWARRAY(char,19+strlen(path)+1); //19: len of " is not a directory"
- strcpy(err,path);
- strcat(err," is not a directory");
- _CLTHROWA_DEL(CL_ERR_IO, err );
+ char* err = _CL_NEWARRAY(char,19+strlen(path)+1); //19: len of " is not a directory"
+ strcpy(err,path);
+ strcat(err," is not a directory");
+ _CLTHROWA_DEL(CL_ERR_IO, err );
}
}
@@ -391,7 +393,7 @@
void FSDirectory::create(){
SCOPED_LOCK_MUTEX(THIS_LOCK)
- struct cl_stat_t fstat;
+ struct cl_stat_t fstat;
if ( fileStat(directory,&fstat) != 0 ) {
//todo: should construct directory using _mkdirs... have to write replacement
if ( _mkdir(directory) == -1 ){
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-29 08:54:17
|
Revision: 2959
http://clucene.svn.sourceforge.net/clucene/?rev=2959&view=rev
Author: ustramooner
Date: 2009-03-29 08:54:12 +0000 (Sun, 29 Mar 2009)
Log Message:
-----------
bug 2670626
Modified Paths:
--------------
branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h
branches/lucene2_3_2/src/core/CLucene/search/SearchHeader.h
branches/lucene2_3_2/src/core/CLucene/util/CLStreams.h
Modified: branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h 2009-03-29 08:45:43 UTC (rev 2958)
+++ branches/lucene2_3_2/src/core/CLucene/queryParser/QueryParser.h 2009-03-29 08:54:12 UTC (rev 2959)
@@ -52,7 +52,7 @@
public:
QueryParserBase(CL_NS(analysis)::Analyzer* analyzer);
- ~QueryParserBase();
+ virtual ~QueryParserBase();
/**
* Whether terms of wildcard, prefix, fuzzy and range queries are to be automatically
Modified: branches/lucene2_3_2/src/core/CLucene/search/SearchHeader.h
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/search/SearchHeader.h 2009-03-29 08:45:43 UTC (rev 2958)
+++ branches/lucene2_3_2/src/core/CLucene/search/SearchHeader.h 2009-03-29 08:54:12 UTC (rev 2959)
@@ -56,7 +56,7 @@
/** Expert: Constructs a TopDocs. TopDocs takes ownership of the ScoreDoc array*/
TopDocs(const int32_t th, ScoreDoc* sds, int32_t scoreDocsLength);
- ~TopDocs();
+ virtual ~TopDocs();
private:
/** Expert: Stores the maximum score value encountered, needed for normalizing. */
Modified: branches/lucene2_3_2/src/core/CLucene/util/CLStreams.h
===================================================================
--- branches/lucene2_3_2/src/core/CLucene/util/CLStreams.h 2009-03-29 08:45:43 UTC (rev 2958)
+++ branches/lucene2_3_2/src/core/CLucene/util/CLStreams.h 2009-03-29 08:54:12 UTC (rev 2959)
@@ -156,7 +156,7 @@
};
class CLUCENE_EXPORT BufferedInputStream: public InputStream, public BufferedStream<signed char>{
public:
- ~BufferedInputStream(){}
+ virtual ~BufferedInputStream(){}
};
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-29 08:45:47
|
Revision: 2958
http://clucene.svn.sourceforge.net/clucene/?rev=2958&view=rev
Author: ustramooner
Date: 2009-03-29 08:45:43 +0000 (Sun, 29 Mar 2009)
Log Message:
-----------
fix for 1948533
Modified Paths:
--------------
trunk/test/CuTest.cpp
trunk/test/analysis/TestAnalysis.cpp
trunk/test/index/TestHighFreqTerms.cpp
trunk/test/index/TestReuters.cpp
trunk/test/search/TestSearch.cpp
trunk/test/store/TestStore.cpp
trunk/test/util/TestPriorityQueue.cpp
Modified: trunk/test/CuTest.cpp
===================================================================
--- trunk/test/CuTest.cpp 2009-03-29 08:44:13 UTC (rev 2957)
+++ trunk/test/CuTest.cpp 2009-03-29 08:45:43 UTC (rev 2958)
@@ -374,7 +374,7 @@
int bufferLen = 25-summary->length-10;
for (int i=0;i<bufferLen;i++ )
CuStringAppend(summary,_T(" "));
- CuStringAppendFormat(summary,_T(" - %dms"),testSuite->timeTaken);
+ CuStringAppendFormat(summary,_T(" - %dms"),(int32_t)testSuite->timeTaken);
}
CuStringAppend(summary, _T("\n"));
}
Modified: trunk/test/analysis/TestAnalysis.cpp
===================================================================
--- trunk/test/analysis/TestAnalysis.cpp 2009-03-29 08:44:13 UTC (rev 2957)
+++ trunk/test/analysis/TestAnalysis.cpp 2009-03-29 08:45:43 UTC (rev 2958)
@@ -24,7 +24,7 @@
uint64_t end = Misc::currentTimeMillis();
int64_t time = end - start;
- CuMessageA (tc,"%d milliseconds to extract ",time);
+ CuMessageA (tc,"%d milliseconds to extract ",(int32_t)time);
CuMessageA (tc,"%d tokens\n", count);
CuMessageA (tc,"%f microseconds/token\n",(time*1000.0)/count );
CuMessageA (tc,"%f megabytes/hour\n", (bytes * 1000.0 * 60.0 * 60.0)/(time * 1000000.0) );
Modified: trunk/test/index/TestHighFreqTerms.cpp
===================================================================
--- trunk/test/index/TestHighFreqTerms.cpp 2009-03-29 08:44:13 UTC (rev 2957)
+++ trunk/test/index/TestHighFreqTerms.cpp 2009-03-29 08:45:43 UTC (rev 2958)
@@ -67,7 +67,7 @@
reader->close();
_CLDELETE( reader );
- //CuMessageA(tc,"%d milliseconds\n",Misc::currentTimeMillis()-start);;
+ //CuMessageA(tc,"%d milliseconds\n",(int32_t)(Misc::currentTimeMillis()-start));
}
void TestHighFreqTerms(CuTest *tc){
char loc[1024];
Modified: trunk/test/index/TestReuters.cpp
===================================================================
--- trunk/test/index/TestReuters.cpp 2009-03-29 08:44:13 UTC (rev 2957)
+++ trunk/test/index/TestReuters.cpp 2009-03-29 08:45:43 UTC (rev 2958)
@@ -176,7 +176,7 @@
diff *= -1;
if ( diff > 16 ){
TCHAR tmp[1024];
- _sntprintf(tmp,1024,_T("Norms are off by more than the threshold! %d, should be %d"), norms2[i], norms1[i]);
+ _sntprintf(tmp,1024,_T("Norms are off by more than the threshold! %d, should be %d"), (int32_t)norms2[i], (int32_t)norms1[i]);
CuAssert(tc,tmp,false);
}
}
Modified: trunk/test/search/TestSearch.cpp
===================================================================
--- trunk/test/search/TestSearch.cpp 2009-03-29 08:44:13 UTC (rev 2957)
+++ trunk/test/search/TestSearch.cpp 2009-03-29 08:45:43 UTC (rev 2958)
@@ -272,7 +272,7 @@
reader->close();
_CLDELETE( reader );
- CuMessageA (tc,"took %d milliseconds\n", Misc::currentTimeMillis()-start);
+ CuMessageA (tc,"took %d milliseconds\n", (int32_t)(Misc::currentTimeMillis()-start));
}
void testNormEncoding(CuTest *tc) {
Modified: trunk/test/store/TestStore.cpp
===================================================================
--- trunk/test/store/TestStore.cpp 2009-03-29 08:44:13 UTC (rev 2957)
+++ trunk/test/store/TestStore.cpp 2009-03-29 08:45:43 UTC (rev 2958)
@@ -52,7 +52,7 @@
file->close();
_CLDELETE(file);
}
- CuMessageA(tc, "%d total milliseconds to create\n", Misc::currentTimeMillis() - start);
+ CuMessageA(tc, "%d total milliseconds to create\n", (int32_t)(Misc::currentTimeMillis() - start));
if (!ram){
store->close();
@@ -83,7 +83,7 @@
_CLDELETE(file);
}
- CuMessageA(tc,"%d total milliseconds to read\n", Misc::currentTimeMillis() - start);
+ CuMessageA(tc,"%d total milliseconds to read\n", (int32_t)(Misc::currentTimeMillis() - start));
srand(1251971);
start = Misc::currentTimeMillis();
@@ -93,8 +93,8 @@
store->deleteFile(name);
}
- CuMessageA(tc, "%d total milliseconds to delete\n",Misc::currentTimeMillis() - start);
- CuMessageA(tc, "%d total milliseconds \n", Misc::currentTimeMillis() - veryStart);
+ CuMessageA(tc, "%d total milliseconds to delete\n",(int32_t)(Misc::currentTimeMillis() - start));
+ CuMessageA(tc, "%d total milliseconds \n", (int32_t)(Misc::currentTimeMillis() - veryStart));
//test makeLock::toString
CL_NS(store)::LuceneLock* lf = store->makeLock("testlock");
Modified: trunk/test/util/TestPriorityQueue.cpp
===================================================================
--- trunk/test/util/TestPriorityQueue.cpp 2009-03-29 08:44:13 UTC (rev 2957)
+++ trunk/test/util/TestPriorityQueue.cpp 2009-03-29 08:45:43 UTC (rev 2958)
@@ -32,7 +32,7 @@
sum += next;
pq.put( next );
}
- CuMessageA(tc,"%d milliseconds/",CL_NS(util)::Misc::currentTimeMillis()-start);
+ CuMessageA(tc,"%d milliseconds/", (int32_t)(CL_NS(util)::Misc::currentTimeMillis()-start));
CuMessageA(tc,"%d puts\n",count);
start = CL_NS(util)::Misc::currentTimeMillis();
@@ -49,7 +49,7 @@
sum2 += last;
}
- CuMessageA(tc,"%d milliseconds",(CL_NS(util)::Misc::currentTimeMillis()-start));
+ CuMessageA(tc,"%d milliseconds", (int32_t)(CL_NS(util)::Misc::currentTimeMillis()-start));
CuMessageA(tc,"/%d pops\n",count);
CLUCENE_ASSERT(sum == sum);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ust...@us...> - 2009-03-29 08:44:17
|
Revision: 2957
http://clucene.svn.sourceforge.net/clucene/?rev=2957&view=rev
Author: ustramooner
Date: 2009-03-29 08:44:13 +0000 (Sun, 29 Mar 2009)
Log Message:
-----------
fix for bug 1948533
Modified Paths:
--------------
branches/lucene2_3_2/src/test/CuTest.cpp
branches/lucene2_3_2/src/test/analysis/TestAnalysis.cpp
branches/lucene2_3_2/src/test/index/TestHighFreqTerms.cpp
branches/lucene2_3_2/src/test/index/TestReuters.cpp
branches/lucene2_3_2/src/test/search/TestSearch.cpp
branches/lucene2_3_2/src/test/search/TestTermVector.cpp
branches/lucene2_3_2/src/test/store/TestStore.cpp
branches/lucene2_3_2/src/test/util/TestPriorityQueue.cpp
Modified: branches/lucene2_3_2/src/test/CuTest.cpp
===================================================================
--- branches/lucene2_3_2/src/test/CuTest.cpp 2009-03-29 08:30:19 UTC (rev 2956)
+++ branches/lucene2_3_2/src/test/CuTest.cpp 2009-03-29 08:44:13 UTC (rev 2957)
@@ -377,7 +377,7 @@
int bufferLen = 25-summary->length-10;
for (int i=0;i<bufferLen;i++ )
CuStringAppend(summary,_T(" "));
- CuStringAppendFormat(summary,_T(" - %dms"),testSuite->timeTaken);
+ CuStringAppendFormat(summary,_T(" - %dms"), (int32_t)testSuite->timeTaken);
}
CuStringAppend(summary, _T("\n"));
}
Modified: branches/lucene2_3_2/src/test/analysis/TestAnalysis.cpp
===================================================================
--- branches/lucene2_3_2/src/test/analysis/TestAnalysis.cpp 2009-03-29 08:30:19 UTC (rev 2956)
+++ branches/lucene2_3_2/src/test/analysis/TestAnalysis.cpp 2009-03-29 08:44:13 UTC (rev 2957)
@@ -24,7 +24,7 @@
uint64_t end = Misc::currentTimeMillis();
int64_t time = end - start;
- CuMessageA (tc,"%d milliseconds to extract ",time);
+ CuMessageA (tc,"%d milliseconds to extract ", (int32_t)time);
CuMessageA (tc,"%d tokens\n", count);
CuMessageA (tc,"%f microseconds/token\n",(time*1000.0)/count );
CuMessageA (tc,"%f megabytes/hour\n", (bytes * 1000.0 * 60.0 * 60.0)/(time * 1000000.0) );
Modified: branches/lucene2_3_2/src/test/index/TestHighFreqTerms.cpp
===================================================================
--- branches/lucene2_3_2/src/test/index/TestHighFreqTerms.cpp 2009-03-29 08:30:19 UTC (rev 2956)
+++ branches/lucene2_3_2/src/test/index/TestHighFreqTerms.cpp 2009-03-29 08:44:13 UTC (rev 2957)
@@ -66,7 +66,7 @@
reader->close();
_CLDELETE( reader );
- //CuMessageA(tc,"%d milliseconds\n",Misc::currentTimeMillis()-start);;
+ //CuMessageA(tc,"%d milliseconds\n",(int32_t)(Misc::currentTimeMillis()-start));
}
void TestHighFreqTerms(CuTest *tc){
char loc[1024];
Modified: branches/lucene2_3_2/src/test/index/TestReuters.cpp
===================================================================
--- branches/lucene2_3_2/src/test/index/TestReuters.cpp 2009-03-29 08:30:19 UTC (rev 2956)
+++ branches/lucene2_3_2/src/test/index/TestReuters.cpp 2009-03-29 08:44:13 UTC (rev 2957)
@@ -181,7 +181,7 @@
diff *= -1;
if ( diff > 16 ){
TCHAR tmp[1024];
- _sntprintf(tmp,1024,_T("Norms are off by more than the threshold! %d, should be %d"), norms2[i], norms1[i]);
+ _sntprintf(tmp,1024,_T("Norms are off by more than the threshold! %d, should be %d"), (int32_t)norms2[i], (int32_t)norms1[i]);
CuAssert(tc,tmp,false);
}
}
Modified: branches/lucene2_3_2/src/test/search/TestSearch.cpp
===================================================================
--- branches/lucene2_3_2/src/test/search/TestSearch.cpp 2009-03-29 08:30:19 UTC (rev 2956)
+++ branches/lucene2_3_2/src/test/search/TestSearch.cpp 2009-03-29 08:44:13 UTC (rev 2957)
@@ -285,7 +285,7 @@
ram->close();
_CLDECDELETE(ram);
- CuMessageA (tc,"took %d milliseconds\n", Misc::currentTimeMillis()-start);
+ CuMessageA (tc,"took %d milliseconds\n", (int32_t)(Misc::currentTimeMillis()-start));
}
void testNormEncoding(CuTest *tc) {
Modified: branches/lucene2_3_2/src/test/search/TestTermVector.cpp
===================================================================
--- branches/lucene2_3_2/src/test/search/TestTermVector.cpp 2009-03-29 08:30:19 UTC (rev 2956)
+++ branches/lucene2_3_2/src/test/search/TestTermVector.cpp 2009-03-29 08:44:13 UTC (rev 2957)
@@ -187,7 +187,7 @@
//float_t coord = sim.coord()
//System.out.println("TF: " + tf + " IDF: " + idf + " LenNorm: " + lNorm);
const TCHAR** vTerms = vector->getTerms();
- const Array<int32_t>* freqs = vector->getTermFrequencies();
+ const ValueArray<int32_t>* freqs = vector->getTermFrequencies();
int32_t i=0;
while ( vTerms && vTerms[i] != NULL )
{
@@ -225,7 +225,7 @@
CLUCENE_ASSERT(vector != NULL);
//_tprintf(_T("Vector: %s\n"),vector);
const TCHAR** terms = vector->getTerms();
- const Array<int32_t>* freqs = vector->getTermFrequencies();
+ const ValueArray<int32_t>* freqs = vector->getTermFrequencies();
CLUCENE_ASSERT(terms != NULL);
int termsLength = 0;
Modified: branches/lucene2_3_2/src/test/store/TestStore.cpp
===================================================================
--- branches/lucene2_3_2/src/test/store/TestStore.cpp 2009-03-29 08:30:19 UTC (rev 2956)
+++ branches/lucene2_3_2/src/test/store/TestStore.cpp 2009-03-29 08:44:13 UTC (rev 2957)
@@ -54,7 +54,7 @@
file->close();
_CLDELETE(file);
}
- CuMessageA(tc, "%d total milliseconds to create\n", Misc::currentTimeMillis() - start);
+ CuMessageA(tc, "%d total milliseconds to create\n", (int32_t)(Misc::currentTimeMillis() - start));
if (!ram){
store->close();
@@ -85,7 +85,7 @@
_CLDELETE(file);
}
- CuMessageA(tc,"%d total milliseconds to read\n", Misc::currentTimeMillis() - start);
+ CuMessageA(tc,"%d total milliseconds to read\n", (int32_t)(Misc::currentTimeMillis() - start));
srand(1251971);
start = Misc::currentTimeMillis();
@@ -95,8 +95,8 @@
store->deleteFile(name);
}
- CuMessageA(tc, "%d total milliseconds to delete\n",Misc::currentTimeMillis() - start);
- CuMessageA(tc, "%d total milliseconds \n", Misc::currentTimeMillis() - veryStart);
+ CuMessageA(tc, "%d total milliseconds to delete\n",(int32_t)(Misc::currentTimeMillis() - start));
+ CuMessageA(tc, "%d total milliseconds \n", (int32_t)(Misc::currentTimeMillis() - veryStart));
//test makeLock::toString
CL_NS(store)::LuceneLock* lf = store->makeLock("testlock");
Modified: branches/lucene2_3_2/src/test/util/TestPriorityQueue.cpp
===================================================================
--- branches/lucene2_3_2/src/test/util/TestPriorityQueue.cpp 2009-03-29 08:30:19 UTC (rev 2956)
+++ branches/lucene2_3_2/src/test/util/TestPriorityQueue.cpp 2009-03-29 08:44:13 UTC (rev 2957)
@@ -33,7 +33,7 @@
sum += next;
pq.put( next );
}
- CuMessageA(tc,"%d milliseconds/",CL_NS(util)::Misc::currentTimeMillis()-start);
+ CuMessageA(tc,"%d milliseconds/", (int32_t)(CL_NS(util)::Misc::currentTimeMillis()-start));
CuMessageA(tc,"%d puts\n",count);
start = CL_NS(util)::Misc::currentTimeMillis();
@@ -50,7 +50,7 @@
sum2 += last;
}
- CuMessageA(tc,"%d milliseconds",(CL_NS(util)::Misc::currentTimeMillis()-start));
+ CuMessageA(tc,"%d milliseconds", (int32_t)(CL_NS(util)::Misc::currentTimeMillis()-start));
CuMessageA(tc,"/%d pops\n",count);
CLUCENE_ASSERT(sum == sum);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
