Menu
▾
▴
clucene-developers — CLucene development-related discussion.
You can subscribe to this list here.
| 2003 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
(16) |
Jul
(56) |
Aug
(2) |
Sep
(62) |
Oct
(71) |
Nov
(45) |
Dec
(6) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2004 |
Jan
(12) |
Feb
(22) |
Mar
|
Apr
(62) |
May
(15) |
Jun
(57) |
Jul
(4) |
Aug
(24) |
Sep
(7) |
Oct
(34) |
Nov
(81) |
Dec
(41) |
| 2005 |
Jan
(70) |
Feb
(51) |
Mar
(46) |
Apr
(16) |
May
(22) |
Jun
(34) |
Jul
(23) |
Aug
(13) |
Sep
(43) |
Oct
(42) |
Nov
(54) |
Dec
(68) |
| 2006 |
Jan
(81) |
Feb
(43) |
Mar
(64) |
Apr
(141) |
May
(37) |
Jun
(101) |
Jul
(112) |
Aug
(32) |
Sep
(85) |
Oct
(63) |
Nov
(84) |
Dec
(81) |
| 2007 |
Jan
(25) |
Feb
(64) |
Mar
(46) |
Apr
(28) |
May
(14) |
Jun
(42) |
Jul
(19) |
Aug
(34) |
Sep
(29) |
Oct
(25) |
Nov
(12) |
Dec
(9) |
| 2008 |
Jan
(15) |
Feb
(34) |
Mar
(37) |
Apr
(23) |
May
(18) |
Jun
(47) |
Jul
(28) |
Aug
(61) |
Sep
(29) |
Oct
(48) |
Nov
(24) |
Dec
(79) |
| 2009 |
Jan
(48) |
Feb
(50) |
Mar
(28) |
Apr
(10) |
May
(51) |
Jun
(22) |
Jul
(125) |
Aug
(29) |
Sep
(38) |
Oct
(29) |
Nov
(58) |
Dec
(32) |
| 2010 |
Jan
(15) |
Feb
(10) |
Mar
(12) |
Apr
(64) |
May
(4) |
Jun
(81) |
Jul
(41) |
Aug
(82) |
Sep
(84) |
Oct
(35) |
Nov
(43) |
Dec
(26) |
| 2011 |
Jan
(59) |
Feb
(25) |
Mar
(23) |
Apr
(14) |
May
(22) |
Jun
(8) |
Jul
(5) |
Aug
(20) |
Sep
(10) |
Oct
(12) |
Nov
(29) |
Dec
(7) |
| 2012 |
Jan
(1) |
Feb
(22) |
Mar
(9) |
Apr
(5) |
May
(2) |
Jun
|
Jul
(6) |
Aug
(2) |
Sep
|
Oct
(5) |
Nov
(9) |
Dec
(10) |
| 2013 |
Jan
(9) |
Feb
(3) |
Mar
(2) |
Apr
(4) |
May
(2) |
Jun
(1) |
Jul
(2) |
Aug
(5) |
Sep
|
Oct
(3) |
Nov
(3) |
Dec
(2) |
| 2014 |
Jan
(1) |
Feb
(2) |
Mar
|
Apr
(10) |
May
(3) |
Jun
|
Jul
|
Aug
|
Sep
(1) |
Oct
|
Nov
|
Dec
(3) |
| 2015 |
Jan
(8) |
Feb
(3) |
Mar
(7) |
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
(1) |
Nov
(3) |
Dec
|
| 2016 |
Jan
(1) |
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
(2) |
| 2018 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
(1) |
Sep
|
Oct
|
Nov
|
Dec
|
| 2019 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
(8) |
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
| 2020 |
Jan
|
Feb
|
Mar
|
Apr
(2) |
May
|
Jun
(1) |
Jul
|
Aug
|
Sep
|
Oct
|
Nov
(1) |
Dec
|
| 2021 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
(1) |
Sep
|
Oct
|
Nov
|
Dec
|
| 2023 |
Jan
|
Feb
|
Mar
(1) |
Apr
|
May
|
Jun
|
Jul
(4) |
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
| 2025 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
(1) |
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
|
From: Stephan B. <sbe...@re...> - 2014-05-08 09:16:47
|
FYI:
-------- Original Message --------
Subject: [Libreoffice-commits] core.git: external/clucene
Date: Thu, 8 May 2014 02:13:15 -0700 (PDT)
From: Stephan Bergmann <sbe...@re...>
Reply-To: lib...@li...
To: lib...@li...
external/clucene/UnpackedTarball_clucene.mk | 1 +
external/clucene/patches/clucene-mutex.patch | 13 +++++++++++++
2 files changed, 14 insertions(+)
New commits:
commit 10314b5d8b653864c92d392cbb774438633b2fe1
Author: Stephan Bergmann <sbe...@re...>
Date: Thu May 8 11:08:51 2014 +0200
CLucene: Helgrind reported "pthread_mutex_destroy of a locked mutex"
> pthread_mutex_destroy
(/usr/src/debug/valgrind-3.9.0/helgrind/hg_intercepts.c:478)
> lucene::util::mutex_thread::~mutex_thread()
(workdir/UnpackedTarball/clucene/src/shared/CLucene/config/threads.cpp:179)
> lucene::store::FSDirectory::FSIndexInput::close()
(workdir/UnpackedTarball/clucene/src/core/CLucene/store/FSDirectory.cpp:225)
> lucene::index::SegmentInfos::read(lucene::store::Directory*, char
const*)
(workdir/UnpackedTarball/clucene/src/core/CLucene/index/SegmentInfos.cpp:770)
>
lucene::index::IndexFileDeleter::IndexFileDeleter(lucene::store::Directory*,
lucene::index::IndexDeletionPolicy*, lucene::index::SegmentInfos*,
std::ostream*, lucene::index::DocumentsWriter*)
(workdir/UnpackedTarball/clucene/src/core/CLucene/index/IndexFileDeleter.cpp:149)
> lucene::index::IndexWriter::init(lucene::store::Directory*,
lucene::analysis::Analyzer*, bool, bool,
lucene::index::IndexDeletionPolicy*, bool)
(workdir/UnpackedTarball/clucene/src/core/CLucene/index/IndexWriter.cpp:262)
> lucene::index::IndexWriter::IndexWriter(char const*,
lucene::analysis::Analyzer*, bool)
(workdir/UnpackedTarball/clucene/src/core/CLucene/index/IndexWriter.cpp:158)
> HelpIndexer::indexDocuments()
(helpcompiler/source/HelpIndexer.cxx:55)
Change-Id: I19cb9bd49b339d206a624c1f1d3dacdd909f4e25
diff --git a/external/clucene/UnpackedTarball_clucene.mk
b/external/clucene/UnpackedTarball_clucene.mk
index 5148fb7..4407f73 100644
--- a/external/clucene/UnpackedTarball_clucene.mk
+++ b/external/clucene/UnpackedTarball_clucene.mk
@@ -31,6 +31,7 @@ $(eval $(call gb_UnpackedTarball_add_patches,clucene,\
external/clucene/patches/clucene-aix.patch \
external/clucene/patches/clucene-git1-win64.patch \
external/clucene/patches/clucene-ub.patch \
+ external/clucene/patches/clucene-mutex.patch \
))
ifneq ($(OS),WNT)
diff --git a/external/clucene/patches/clucene-mutex.patch
b/external/clucene/patches/clucene-mutex.patch
new file mode 100644
index 0000000..2a6b7d2
--- /dev/null
+++ b/external/clucene/patches/clucene-mutex.patch
@@ -0,0 +1,13 @@
+--- src/core/CLucene/store/FSDirectory.cpp
++++ src/core/CLucene/store/FSDirectory.cpp
+@@ -219,9 +219,8 @@
+ _CLDECDELETE(handle);
+
+ //printf("handle=%d\n", handle->__cl_refcount);
+- if ( dounlock ){
+ mutex->unlock();
+- }else{
++ if ( !dounlock ){
+ delete mutex;
+ }
+ }
|
|
From: <Rog...@su...> - 2014-05-05 11:15:27
|
Hello,
There is a crash in IndexWrite.cpp IndexWrite::flushDocStores()
I have gone through some searches in the Lucene pages. It is mentioned to be a problem but no fix is visible.
Could you confirm the following fix I have done below.
Thanks
Roger Bosshard
In IndexWrite.cpp
bool IndexWriter::flushDocStores() {
SCOPED_LOCK_MUTEX(THIS_LOCK)
// Roger: 05.05.2014
// If index writer useCompoundDocStore, then files will be deleted in closeDocStore()
// Later on, CompoundFileWriter would crash as files is not valid anymore
// To prevent this, unlink _files from docWriter if the index writer is useCompoundDocStore
//const std::vector<std::string>& files = docWriter->files();
std::vector<std::string>* filesptr = docWriter->filesptr(mergePolicy->useCompoundDocStore(segmentInfos));
const std::vector<std::string>& files = *filesptr;
bool useCompoundDocStore = false;
if (files.size() > 0) {
string docStoreSegment;
bool success = false;
try {
docStoreSegment = docWriter->closeDocStore();
success = true;
} _CLFINALLY (
if (!success) {
if (infoStream != NULL)
message(string("hit exception closing doc store segment"));
docWriter->abort(NULL);
}
)
useCompoundDocStore = mergePolicy->useCompoundDocStore(segmentInfos);
if (useCompoundDocStore && !docStoreSegment.empty()) {
// Now build compound doc store file
success = false;
const int32_t numSegments = segmentInfos->size();
const string compoundFileName = docStoreSegment + "." + IndexFileNames::COMPOUND_FILE_STORE_EXTENSION;
try {
CompoundFileWriter cfsWriter(directory, compoundFileName.c_str());
const size_t size = files.size();
for(size_t i=0;i<size;++i)
cfsWriter.addFile(files[i].c_str());
// Perform the merge
cfsWriter.close();
for(int32_t i=0;i<numSegments;i++) {
SegmentInfo* si = segmentInfos->info(i);
if (si->getDocStoreOffset() != -1 &&
si->getDocStoreSegment().compare(docStoreSegment)==0)
si->setDocStoreIsCompoundFile(true);
}
checkpoint();
success = true;
} _CLFINALLY (
if (!success) {
if (infoStream != NULL)
message("hit exception building compound file doc store for segment " + docStoreSegment);
// Rollback to no compound file
for(int32_t i=0;i<numSegments;i++) {
SegmentInfo* si = segmentInfos->info(i);
if (si->getDocStoreOffset() != -1 &&
si->getDocStoreSegment().compare(docStoreSegment)==0 )
si->setDocStoreIsCompoundFile(false);
}
deleter->deleteFile(compoundFileName.c_str());
deletePartialSegmentsFile();
}
)
deleter->checkpoint(segmentInfos, false);
}
if (useCompoundDocStore)
_CLLDELETE(filesptr); // Roger: 05.05.2014: delete if it has been unlinked
}
return useCompoundDocStore;
}
**
In DocumentWriter.cpp
std::vector<std::string>* DocumentsWriter::filesptr(bool unlink) {
SCOPED_LOCK_MUTEX(THIS_LOCK)
files(); // Build _files if not already built
if (unlink) {
std::vector<std::string>* unlinkptr = _files;
_files = NULL; // unlink; caller must free ptr
return unlinkptr;
}
else
return _files;
}
In _DocumentWrite.h
/* Returns list of files in use by this instance,
* including any flushed segments. */
const std::vector<std::string>& files();
std::vector<std::string>* filesptr(bool unlink);
|
|
From: Ahmed S. <ci7...@gm...> - 2014-04-30 13:45:16
|
<html style="direction: ltr;">
<head>
<meta content="text/html; charset=ISO-8859-1"
http-equiv="Content-Type">
<style type="text/css">body p { margin-bottom: 0cm; margin-top: 0pt; } </style>
</head>
<body style="direction: ltr;"
bidimailui-detected-decoding-type="latin-charset" bgcolor="#FFFFFF"
text="#000000">
<div class="moz-cite-prefix">Hi<br>
<br>
You can get the recent code change from git repo at:<br>
<a class="moz-txt-link-freetext" href="http://sourceforge.net/p/clucene/code">http://sourceforge.net/p/clucene/code</a><br>
<br>
Ahmed<br>
<br>
Le 30/04/2014 13:31, <a class="moz-txt-link-abbreviated" href="mailto:Rog...@su...">Rog...@su...</a> a écrit :<br>
</div>
<blockquote
cite="mid:F8F...@em...rp"
type="cite">
<pre wrap="">I've downloaded 2.3.3.4 from SourceForce recently (a zip). IndexWrite.cpp is dated 17.03.2011 00:21
Is there any better I have to look at ?
Thanks
Roger
-----Original Message-----
From: <a class="moz-txt-link-abbreviated" href="mailto:clu...@li...">clu...@li...</a> [<a class="moz-txt-link-freetext" href="mailto:clu...@li...">mailto:clu...@li...</a>]
Sent: mercredi 30 avril 2014 14:18
To: <a class="moz-txt-link-abbreviated" href="mailto:clu...@li...">clu...@li...</a>
Subject: CLucene-developers Digest, Vol 85, Issue 4
Send CLucene-developers mailing list submissions to
<a class="moz-txt-link-abbreviated" href="mailto:clu...@li...">clu...@li...</a>
To subscribe or unsubscribe via the World Wide Web, visit
<a class="moz-txt-link-freetext" href="https://lists.sourceforge.net/lists/listinfo/clucene-developers">https://lists.sourceforge.net/lists/listinfo/clucene-developers</a>
or, via email, send a message with subject or body 'help' to
<a class="moz-txt-link-abbreviated" href="mailto:clu...@li...">clu...@li...</a>
You can reach the person managing the list at
<a class="moz-txt-link-abbreviated" href="mailto:clu...@li...">clu...@li...</a>
When replying, please edit your Subject line so it is more specific than "Re: Contents of CLucene-developers digest..."
Today's Topics:
1. Re: Memory leak in IndexWriter::applyDeletes (Kostka Bo?ivoj)
----------------------------------------------------------------------
Message: 1
Date: Wed, 30 Apr 2014 12:17:25 +0000
From: Kostka Bo?ivoj <a class="moz-txt-link-rfc2396E" href="mailto:ko...@to..."><ko...@to...></a>
Subject: Re: [CLucene-dev] Memory leak in IndexWriter::applyDeletes
To: <a class="moz-txt-link-rfc2396E" href="mailto:clu...@li...">"clu...@li..."</a>
<a class="moz-txt-link-rfc2396E" href="mailto:clu...@li..."><clu...@li...></a>
Message-ID:
<a class="moz-txt-link-rfc2396E" href="mailto:0BE...@TV...kd.local"><0BE...@TV...kd.local></a>
Content-Type: text/plain; charset="utf-8"
Are you sure you have last code. This is already fixed in master branch
From: <a class="moz-txt-link-abbreviated" href="mailto:Rog...@su...">Rog...@su...</a> [<a class="moz-txt-link-freetext" href="mailto:Rog...@su...">mailto:Rog...@su...</a>]
Sent: Tuesday, April 29, 2014 6:02 PM
To: <a class="moz-txt-link-abbreviated" href="mailto:clu...@li...">clu...@li...</a>
Subject: [CLucene-dev] Memory leak in IndexWriter::applyDeletes
Hello,
I think there is a leak in the file IndexWrite.cpp, function IndexWriter::applyDeletes
for (int32_t i = 0; i < infosEnd; i++) {
IndexReader* reader = NULL;
try {
reader = SegmentReader::get(segmentInfos->info(i), false);
// Apply delete terms to disk segments
// except the one just flushed from ram.
_internal->applyDeletes(bufferedDeleteTerms, reader);
} _CLFINALLY (
if (reader != NULL) {
try {
reader->doCommit();
} _CLFINALLY (
reader->doClose();
_CLLDELETE(reader); // 17.04.2014, memory leak fix; reader was not deleted
)
}
)
}
// Clean up bufferedDeleteTerms.
docWriter->clearBufferedDeletes();
}
Regards
Roger Bosshard
Roger Bosshard ? Head of Development ? MarketMap Realtime ? SunGard ? Route Fran?ois-Peyrot 12 ? 1218 Le Grand Saconnex, Geneva, Switzerland Tel +41 22 715 44 52 * Fax +41 22 715 44 11 * <a class="moz-txt-link-abbreviated" href="http://www.sungard.com/marketmap">www.sungard.com/marketmap</a><a class="moz-txt-link-rfc2396E" href="http://www.sungard.com/marketmap"><http://www.sungard.com/marketmap></a>
[Description: Description: Description: Description: coc-signature-03-2012]<a class="moz-txt-link-rfc2396E" href="http://www.capitalize-on-change.com/?email=70150000000Y1Et"><http://www.capitalize-on-change.com/?email=70150000000Y1Et></a>
Join the online conversation with SunGard?s customers, partners and industry experts and find an event near you at: <a class="moz-txt-link-abbreviated" href="http://www.sungard.com/ten">www.sungard.com/ten</a><a class="moz-txt-link-rfc2396E" href="http://www.capitalize-on-change.com/?email=70150000000Y1Et"><http://www.capitalize-on-change.com/?email=70150000000Y1Et></a>.
CONFIDENTIALITY: This e-mail (including any attachments) may contain confidential, proprietary and privileged information, and unauthorized disclosure or use is prohibited. If you receive this e-mail in error, please notify the sender and delete this e-mail from your system.
-------------- next part --------------
An HTML attachment was scrubbed...
-------------- next part --------------
A non-text attachment was scrubbed...
Name: image001.gif
Type: image/gif
Size: 8696 bytes
Desc: image001.gif
------------------------------
------------------------------------------------------------------------------
"Accelerate Dev Cycles with Automated Cross-Browser Testing - For FREE Instantly run your Selenium tests across 300+ browser/OS combos. Get unparalleled scalability from the best Selenium testing platform available.
Simple to use. Nothing to install. Get started now for free.<a class="moz-txt-link-rfc2396E" href="http://p.sf.net/sfu/SauceLabs------------------------------_________________________________...@li...https://lists.sourceforge.net/lists/listinfo/clucene-developersEndofCLucene-developersDigest,Vol85,Issue4*************************************************------------------------------------------------------------------------------">"
http://p.sf.net/sfu/SauceLabs
------------------------------
_______________________________________________
CLucene-developers mailing list
CLu...@li...
https://lists.sourceforge.net/lists/listinfo/clucene-developers
End of CLucene-developers Digest, Vol 85, Issue 4
*************************************************
------------------------------------------------------------------------------
"</a>Accelerate Dev Cycles with Automated Cross-Browser Testing - For FREE
Instantly run your Selenium tests across 300+ browser/OS combos. Get
unparalleled scalability from the best Selenium testing platform available.
Simple to use. Nothing to install. Get started now for free."
<a class="moz-txt-link-freetext" href="http://p.sf.net/sfu/SauceLabs">http://p.sf.net/sfu/SauceLabs</a>
_______________________________________________
CLucene-developers mailing list
<a class="moz-txt-link-abbreviated" href="mailto:CLu...@li...">CLu...@li...</a>
<a class="moz-txt-link-freetext" href="https://lists.sourceforge.net/lists/listinfo/clucene-developers">https://lists.sourceforge.net/lists/listinfo/clucene-developers</a>
</pre>
</blockquote>
<br>
</body>
</html>
|
|
From: <Rog...@su...> - 2014-04-30 12:31:40
|
I've downloaded 2.3.3.4 from SourceForce recently (a zip). IndexWrite.cpp is dated 17.03.2011 00:21 Is there any better I have to look at ? Thanks Roger -----Original Message----- From: clu...@li... [mailto:clu...@li...] Sent: mercredi 30 avril 2014 14:18 To: clu...@li... Subject: CLucene-developers Digest, Vol 85, Issue 4 Send CLucene-developers mailing list submissions to clu...@li... To subscribe or unsubscribe via the World Wide Web, visit https://lists.sourceforge.net/lists/listinfo/clucene-developers or, via email, send a message with subject or body 'help' to clu...@li... You can reach the person managing the list at clu...@li... When replying, please edit your Subject line so it is more specific than "Re: Contents of CLucene-developers digest..." Today's Topics: 1. Re: Memory leak in IndexWriter::applyDeletes (Kostka Bo?ivoj) ---------------------------------------------------------------------- Message: 1 Date: Wed, 30 Apr 2014 12:17:25 +0000 From: Kostka Bo?ivoj <ko...@to...> Subject: Re: [CLucene-dev] Memory leak in IndexWriter::applyDeletes To: "clu...@li..." <clu...@li...> Message-ID: <0BE...@TV...kd.local> Content-Type: text/plain; charset="utf-8" Are you sure you have last code. This is already fixed in master branch From: Rog...@su... [mailto:Rog...@su...] Sent: Tuesday, April 29, 2014 6:02 PM To: clu...@li... Subject: [CLucene-dev] Memory leak in IndexWriter::applyDeletes Hello, I think there is a leak in the file IndexWrite.cpp, function IndexWriter::applyDeletes for (int32_t i = 0; i < infosEnd; i++) { IndexReader* reader = NULL; try { reader = SegmentReader::get(segmentInfos->info(i), false); // Apply delete terms to disk segments // except the one just flushed from ram. _internal->applyDeletes(bufferedDeleteTerms, reader); } _CLFINALLY ( if (reader != NULL) { try { reader->doCommit(); } _CLFINALLY ( reader->doClose(); _CLLDELETE(reader); // 17.04.2014, memory leak fix; reader was not deleted ) } ) } // Clean up bufferedDeleteTerms. docWriter->clearBufferedDeletes(); } Regards Roger Bosshard Roger Bosshard ? Head of Development ? MarketMap Realtime ? SunGard ? Route Fran?ois-Peyrot 12 ? 1218 Le Grand Saconnex, Geneva, Switzerland Tel +41 22 715 44 52 * Fax +41 22 715 44 11 * www.sungard.com/marketmap<http://www.sungard.com/marketmap> [Description: Description: Description: Description: coc-signature-03-2012]<http://www.capitalize-on-change.com/?email=70150000000Y1Et> Join the online conversation with SunGard?s customers, partners and industry experts and find an event near you at: www.sungard.com/ten<http://www.capitalize-on-change.com/?email=70150000000Y1Et>. CONFIDENTIALITY: This e-mail (including any attachments) may contain confidential, proprietary and privileged information, and unauthorized disclosure or use is prohibited. If you receive this e-mail in error, please notify the sender and delete this e-mail from your system. -------------- next part -------------- An HTML attachment was scrubbed... -------------- next part -------------- A non-text attachment was scrubbed... Name: image001.gif Type: image/gif Size: 8696 bytes Desc: image001.gif ------------------------------ ------------------------------------------------------------------------------ "Accelerate Dev Cycles with Automated Cross-Browser Testing - For FREE Instantly run your Selenium tests across 300+ browser/OS combos. Get unparalleled scalability from the best Selenium testing platform available. Simple to use. Nothing to install. Get started now for free." http://p.sf.net/sfu/SauceLabs ------------------------------ _______________________________________________ CLucene-developers mailing list CLu...@li... https://lists.sourceforge.net/lists/listinfo/clucene-developers End of CLucene-developers Digest, Vol 85, Issue 4 ************************************************* |
|
From: Kostka B. <ko...@to...> - 2014-04-30 07:28:19
|
Hi, You can share one index between any number of processes, which have index opened for reading. You can also have one process which writes to the index at the same time. If you add/delete some documents, you have to create new IndexReader in running searching processes to see the changes You can use isCurrent method to check if index was changed. Hope this help Borek From: Max . [mailto:ya...@li...] Sent: Wednesday, April 23, 2014 10:55 PM To: clu...@li... Subject: [CLucene-dev] Multi-process access Hi, I am setting up a search engine based on CLucene which saved my life so thank you guys :) ! The idea is to have multiple servers do the indexing and other servers handle searching, I use a conventional round robin broker for that. Each one of my client will have its own index, say bob connects to our web interface and searches for "foo" it will look it up in bob.idx so I will actually have lots of indexes. I see two approach to this : 1) Have one index file on a NAS and everyone taps into that but I don't know how safe this is in terms of resource conflicts ? 2) Have indexers work on their own write-only index and replicate the index on search servers which will have read-only indexes and be put to sleep while the index is being replicated. Can I have your opinions on approaching this problem ? Thanks, Max |
|
From: Max . <ya...@li...> - 2014-04-23 20:55:07
|
Hi, I am setting up a search engine based on CLucene which saved my life so thank you guys :) ! The idea is to have multiple servers do the indexing and other servers handle searching, I use a conventional round robin broker for that. Each one of my client will have its own index, say bob connects to our web interface and searches for "foo" it will look it up in bob.idx so I will actually have lots of indexes. I see two approach to this : 1) Have one index file on a NAS and everyone taps into that but I don't know how safe this is in terms of resource conflicts ? 2) Have indexers work on their own write-only index and replicate the index on search servers which will have read-only indexes and be put to sleep while the index is being replicated. Can I have your opinions on approaching this problem ? Thanks, Max |
|
From: Kostka B. <ko...@to...> - 2014-04-23 11:43:34
|
Answer is probably yes. I'm still fixing bugs time to time but nobody else seems to be active here From: Ahmed Saidi [mailto:ci7...@gm...] Sent: Saturday, April 19, 2014 9:08 AM To: clu...@li... Subject: [CLucene-dev] Is clucene dead? Hi, When i started working with CLucene in 2011 the development and mailing list was active, after the announcement of Lucene++ this project became inactive. So, is this project *dead*? Thank you for all the effort and time you spent to make this project as it's now, i know it cover everything you need for indexing/searching but we want more :) Ahmed |
|
From: Ahmed S. <ci7...@gm...> - 2014-04-19 07:07:58
|
<html style="direction: ltr;">
<head>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1">
<style type="text/css">body p { margin-bottom: 0cm; margin-top: 0pt; } </style>
</head>
<body style="direction: ltr;"
bidimailui-detected-decoding-type="latin-charset" bgcolor="#FFFFFF"
text="#000000">
Hi,<br>
<br>
When i started working with CLucene in 2011 the development and
mailing list was active, after the announcement of Lucene++ this
project became inactive.<br>
So, is this project *dead*?<br>
<br>
Thank you for all the effort and time you spent to make this project
as it's now, i know it cover everything you need for
indexing/searching but we want more :)<br>
<br>
Ahmed<br>
</body>
</html>
|
|
From: Matt M. <cl...@ms...> - 2014-04-08 19:47:05
|
Hello, I've been using CLucene for some time on various projects, and really appreciate all the work that has gone into it. So thank you to all that have contributed to date! I've also noticed some warnings that clang emits when compiling the code that I hoped to fix. I've got a fork of the code over on GitHub and here are two sets of changes that I hoped might be useful, and that are logically correct. https://github.com/Blue-Rocket/clucene/compare/develop?expand=1 Cheers, Matt |
|
From: Kostka B. <ko...@to...> - 2014-02-16 18:08:39
|
ZLIB is a compression library used in clucene. Although some zlib source code is enclosed in src/ext/zlib I recommend to download from original location and build it before you build clucene. See http://www.zlib.net/ for details You can also download windows binaries, if you don't want to build from source e.g. From http://gnuwin32.sourceforge.net/packages/zlib.htm > -----Original Message----- > From: Imran Rafique [mailto:mai...@ya...] > Sent: Friday, February 14, 2014 6:17 PM > To: clu...@li... > Subject: Re: [CLucene-dev] pre-built clucene windows binaries (32 and > 64 bit) for download ? > > Hi. I could not fine the option ZLIB_INCLUDE_DIR at step 10. Please > help me out. Thanks > > > ----------------------------------------------------------------------- > ------- > Android apps run on BlackBerry 10 > Introducing the new BlackBerry 10.2.1 Runtime for Android apps. > Now with support for Jelly Bean, Bluetooth, Mapview and more. > Get your Android app in front of a whole new audience. Start now. > http://pubads.g.doubleclick.net/gampad/clk?id=124407151&iu=/4140/ostg.c > lktrk > _______________________________________________ > CLucene-developers mailing list > CLu...@li... > https://lists.sourceforge.net/lists/listinfo/clucene-developers |
|
From: Imran R. <mai...@ya...> - 2014-02-15 14:15:12
|
Hi. I could not fine the option ZLIB_INCLUDE_DIR at step 10. Please help me out. Thanks |
|
From: Adrian J. <Adr...@io...> - 2014-01-13 15:50:28
|
Hi Borek, Thanks for your reply. This seems to be the issue we are having - CLucene is NOT restricting write access to the index (i.e. it seems it is not handling multi-threading correctly). When we launch, for example, 2 processes simultaneously, the 2nd one crashes when trying to access the index (it seems as if the 1st one is not releasing the lock on the directory). Unfortunately, we can't add synchronisation code as we are shipping a version of CLucene which is integrated with Qt 4.7.3. Qt support has been unable to help so far so I was hoping there would be other CLucene users who have seen this behaviour and may have a fix/workaround for it. Thanks, Adrian > Hi, > > In CLucene only one process can write to the index at the same time. As far as I know it is by intent. > The only solution is to synchronize the write access in your apllication (eg. by boost::file_lock) > > Regards > > Borek > >> -----Original Message----- >> From: Adrian Jordan [mailto:Adr...@io...] >> Sent: Thursday, December 19, 2013 12:31 PM >> To: clu...@li... >> Subject: [CLucene-dev] IndexWriter concurrency >> >> Hello, >> >> We are using Qt Help (version 4.7.3) to provide documentation for our >> application. >> >> When we launch it, Qt creates an index directory based upon the Qt Help >> collection file, which is used by CLucene (bundled with the Qt Help >> Framework) to index the help contents. >> >> So we have for example: >> >> ../documentation/ >> help.qhc >> .help/ >> segments >> _6d.cfs >> deletable >> >> This setup is not working in our application, which consists of >> multiple processes running concurrently, as it causes one or more >> processes to crash when we startup. >> >> >From the GDB back trace from the core file that was produced, the >> CLucene code is crashing in: >> >> RAMDirectory::close (line 352) >> files.clear(); >> TransactionalRAMDirectory::close (line 209) >> RAMDirectory::close(); >> IndexWriter::close (line 165) >> ramDirectory->close(); >> >> This seems to point to an issue with concurrent access to the >> IndexWriter (related to SCOPED_LOCK_MUTEX? VoidMap?). I believe CLucene >> uses a write lock on this directory (.help in our case) that can only >> be access by one process at a time. >> >> We can reproduce it repeatedly be removing the files under the .help >> directory and restarting our application. >> >> Upgrading to the latest version of CLucene didn't resolve the issue (I >> can't even find out which version we are using!). >> >> So: >> >> 1/ Should CLucene support multiple processes/instances of the same >> process writing to this index directory? >> 2/ Is there setting we can use to try and resolve this (e.g increase >> the write lock timeout)? >> 3/ Is this a known issue and has it been fixed? >> >> I look forward to hearing from anyone with any ideas about this. >> >> Thanks, >> >> Adrian >> >> ________________________________ >> >> >> This email and any files transmitted with it are confidential and are >> intended solely for the use of the individual or entity to whom they >> are addressed. If you are not the original recipient or the person >> responsible for delivering the email to the intended recipient, be >> advised that you have received this email in error, and that any use, >> dissemination, forwarding, printing, or copying of this email is >> strictly prohibited. If you received this email in error, please >> immediately notify the sender and delete the original. >> >> >> ----------------------------------------------------------------------- >> ------- >> Rapidly troubleshoot problems before they affect your business. Most IT >> organizations don't have a clear picture of how application performance >> affects their revenue. With AppDynamics, you get 100% visibility into >> your Java,.NET, & PHP application. Start your 15-day FREE TRIAL of >> AppDynamics Pro! >> http://pubads.g.doubleclick.net/gampad/clk?id=84349831&iu=/4140/ostg.cl >> ktrk >> _______________________________________________ >> CLucene-developers mailing list >> CLu...@li... >> https://lists.sourceforge.net/lists/listinfo/clucene-developers > ------------------------------------------------------------------------------ > Rapidly troubleshoot problems before they affect your business. Most IT > organizations don't have a clear picture of how application performance > affects their revenue. With AppDynamics, you get 100% visibility into your > Java,.NET, & PHP application. Start your 15-day FREE TRIAL of AppDynamics Pro! > http://pubads.g.doubleclick.net/gampad/clk?id=84349831&iu=/4140/ostg.clktrk > _______________________________________________ > CLucene-developers mailing list > CLu...@li... > https://lists.sourceforge.net/lists/listinfo/clucene-developers ________________________________ This email and any files transmitted with it are confidential and are intended solely for the use of the individual or entity to whom they are addressed. If you are not the original recipient or the person responsible for delivering the email to the intended recipient, be advised that you have received this email in error, and that any use, dissemination, forwarding, printing, or copying of this email is strictly prohibited. If you received this email in error, please immediately notify the sender and delete the original. |
|
From: Kostka B. <ko...@to...> - 2013-12-30 13:49:23
|
Hi, In CLucene only one process can write to the index at the same time. As far as I know it is by intent. The only solution is to synchronize the write access in your apllication (eg. by boost::file_lock) Regards Borek > -----Original Message----- > From: Adrian Jordan [mailto:Adr...@io...] > Sent: Thursday, December 19, 2013 12:31 PM > To: clu...@li... > Subject: [CLucene-dev] IndexWriter concurrency > > Hello, > > We are using Qt Help (version 4.7.3) to provide documentation for our > application. > > When we launch it, Qt creates an index directory based upon the Qt Help > collection file, which is used by CLucene (bundled with the Qt Help > Framework) to index the help contents. > > So we have for example: > > ../documentation/ > help.qhc > .help/ > segments > _6d.cfs > deletable > > This setup is not working in our application, which consists of > multiple processes running concurrently, as it causes one or more > processes to crash when we startup. > > >From the GDB back trace from the core file that was produced, the > CLucene code is crashing in: > > RAMDirectory::close (line 352) > files.clear(); > TransactionalRAMDirectory::close (line 209) > RAMDirectory::close(); > IndexWriter::close (line 165) > ramDirectory->close(); > > This seems to point to an issue with concurrent access to the > IndexWriter (related to SCOPED_LOCK_MUTEX? VoidMap?). I believe CLucene > uses a write lock on this directory (.help in our case) that can only > be access by one process at a time. > > We can reproduce it repeatedly be removing the files under the .help > directory and restarting our application. > > Upgrading to the latest version of CLucene didn't resolve the issue (I > can't even find out which version we are using!). > > So: > > 1/ Should CLucene support multiple processes/instances of the same > process writing to this index directory? > 2/ Is there setting we can use to try and resolve this (e.g increase > the write lock timeout)? > 3/ Is this a known issue and has it been fixed? > > I look forward to hearing from anyone with any ideas about this. > > Thanks, > > Adrian > > ________________________________ > > > This email and any files transmitted with it are confidential and are > intended solely for the use of the individual or entity to whom they > are addressed. If you are not the original recipient or the person > responsible for delivering the email to the intended recipient, be > advised that you have received this email in error, and that any use, > dissemination, forwarding, printing, or copying of this email is > strictly prohibited. If you received this email in error, please > immediately notify the sender and delete the original. > > > ----------------------------------------------------------------------- > ------- > Rapidly troubleshoot problems before they affect your business. Most IT > organizations don't have a clear picture of how application performance > affects their revenue. With AppDynamics, you get 100% visibility into > your Java,.NET, & PHP application. Start your 15-day FREE TRIAL of > AppDynamics Pro! > http://pubads.g.doubleclick.net/gampad/clk?id=84349831&iu=/4140/ostg.cl > ktrk > _______________________________________________ > CLucene-developers mailing list > CLu...@li... > https://lists.sourceforge.net/lists/listinfo/clucene-developers |
|
From: Adrian J. <Adr...@io...> - 2013-12-19 11:31:14
|
Hello,
We are using Qt Help (version 4.7.3) to provide documentation for our application.
When we launch it, Qt creates an index directory based upon the Qt Help collection file, which is used by CLucene (bundled with the Qt Help Framework) to index the help contents.
So we have for example:
../documentation/
help.qhc
.help/
segments
_6d.cfs
deletable
This setup is not working in our application, which consists of multiple processes running concurrently, as it causes one or more processes to crash when we startup.
>From the GDB back trace from the core file that was produced, the CLucene code is crashing in:
RAMDirectory::close (line 352) files.clear();
TransactionalRAMDirectory::close (line 209) RAMDirectory::close();
IndexWriter::close (line 165) ramDirectory->close();
This seems to point to an issue with concurrent access to the IndexWriter (related to SCOPED_LOCK_MUTEX? VoidMap?). I believe CLucene uses a write lock on this directory (.help in our case) that can only be access by one process at a time.
We can reproduce it repeatedly be removing the files under the .help directory and restarting our application.
Upgrading to the latest version of CLucene didn't resolve the issue (I can't even find out which version we are using!).
So:
1/ Should CLucene support multiple processes/instances of the same process writing to this index directory?
2/ Is there setting we can use to try and resolve this (e.g increase the write lock timeout)?
3/ Is this a known issue and has it been fixed?
I look forward to hearing from anyone with any ideas about this.
Thanks,
Adrian
________________________________
This email and any files transmitted with it are confidential and are intended solely for the use of the individual or entity to whom they are addressed. If you are not the original recipient or the person responsible for delivering the email to the intended recipient, be advised that you have received this email in error, and that any use, dissemination, forwarding, printing, or copying of this email is strictly prohibited. If you received this email in error, please immediately notify the sender and delete the original.
|
|
From: Kostka B. <ko...@to...> - 2013-11-20 22:18:03
|
Hi, Although this is relatively simple for single term (using TermEnum and TermDocs) I think there is no easy way how to do that for phrase. Perhaps you can use Highlighter from contrib-libs, but it is probably not very efficient way. Borek From: Jonas Poelmans [mailto:jon...@gm...] Sent: Wednesday, November 20, 2013 8:49 PM To: clu...@li... Subject: Re: [CLucene-dev] CLucene Questions Dear all My apologies for once more disturbing. After a deep search in the API and test code of CLucene I managed to answer 2 of my 3 previous questions. However this one I have not been able to solve: When building a query and applying it to an IndexSearcher, is it possible to obtain for each document the number of times the query phrase was found in the document. For example when looking for the phrase "cat is walking" , I would like to know for each document how many times it was found. Is this possible and if yes, how? Thanks Jonas |
|
From: Jonas P. <jon...@gm...> - 2013-11-20 19:48:47
|
Dear all My apologies for once more disturbing. After a deep search in the API and test code of CLucene I managed to answer 2 of my 3 previous questions. However this one I have not been able to solve: When building a query and applying it to an IndexSearcher, is it possible to obtain for each document the number of times the query phrase was found in the document. For example when looking for the phrase "cat is walking" , I would like to know for each document how many times it was found. Is this possible and if yes, how? Thanks Jonas |
|
From: Jonas P. <jon...@gm...> - 2013-11-13 19:51:34
|
Dear all I managed to compile CLucene and integrate it with my application. I managed to get all basic functionality working for version 2.3.3.4. I have however three questions for which I didn't find an answer in the documentation. I would be grateful if somebody could answer some or all of these questions: - When building a query and applying it to an IndexSearcher, is it possible to obtain for each document the number of times the query phrase was found in the document. For example when looking for the phrase "cat is walking" , I would like to know for each document how many times it was found. Is this possible and if yes, how? - I didn't manage to create a MultiReader from an existing IndexReader. I need a MultiReader because I need to use the method getTermFreqVector() which is purely virtual in IndexReader. Is it possible to show this by some sample code? - When I build an index several times after each other on excactly the same text collection, the number of terms in the index changes all the time. Why does this happen? Are the analyzers using some heuristics? Kind regards Jonas |
|
From: Kostka B. <ko...@to...> - 2013-10-09 18:46:39
|
You should pass dynamically allocated Field object, as you can see e.g. in TestIndexWriter code:
doc.add ( *_CLNEW Field(_T("field0"),fld,Field::STORE_YES | Field::INDEX_TOKENIZED) );
Hope this helps
Borek
From: Michael Zhang (BLOOMBERG/ 731 LEXIN) [mailto:mzh...@bl...]
Sent: Wednesday, October 09, 2013 4:38 PM
To: clu...@li...
Subject: [CLucene-dev] crash -- R6025 pure virtual function call
I try to write a test application in Microsoft VS 2010 as following.
and when I debug the code, it crash at:
writer.addDocument(&doc);
and I get an error message, it shows:
Runtime Error!
Program: C:...
R6025
- pure virtual function call
Anybody knows how to fix it?
Thanks.
___________________________________________________________________
RAMDirectory directory;
StandardAnalyzer analyzer;
IndexWriter writer(&directory, &analyzer, true);
lucene::document::Document doc;
doc.add(lucene::document::Field(L"content", L"abcd", lucene::document::Field::STORE_YES| lucene::document::Field::INDEX_TOKENIZED, true));
writer.addDocument(&doc);
writer.optimize();
writer.close();
|
|
From: Michael Z. (B. 7. LEXIN) <mzh...@bl...> - 2013-10-09 18:37:33
|
Hi Borek,
Yes, it works.
Thanks a lot.
BTW, do I need delete Field to free the memory?
Thanks a lot for your reply and help. wish you have a great day.
Yours,
Michael zhang
----- Original Message -----
From: ko...@to...
To: MICHAEL ZHANG (BLOOMBERG/ 731 LEXIN), clu...@li...
At: Oct 9 2013 14:34:05
You should pass dynamically allocated Field object, as you can see e.g. in TestIndexWriter code:
doc.add ( *_CLNEW Field(_T("field0"),fld,Field::STORE_YES | Field::INDEX_TOKENIZED) );
Hope this helps
Borek
From: Michael Zhang (BLOOMBERG/ 731 LEXIN) [mailto:mzh...@bl...]
Sent: Wednesday, October 09, 2013 4:38 PM
To: clu...@li...
Subject: [CLucene-dev] crash -- R6025 pure virtual function call
I try to write a test application in Microsoft VS 2010 as following.
and when I debug the code, it crash at:
writer.addDocument(&doc);
and I get an error message, it shows:
Runtime Error!
Program: C:...
R6025
- pure virtual function call
Anybody knows how to fix it?
Thanks.
___________________________________________________________________
RAMDirectory directory;
StandardAnalyzer analyzer;
IndexWriter writer(&directory, &analyzer, true);
lucene::document::Document doc;
doc.add(lucene::document::Field(L"content", L"abcd", lucene::document::Field::STORE_YES| lucene::document::Field::INDEX_TOKENIZED, true));
writer.addDocument(&doc);
writer.optimize();
writer.close();
|
|
From: Michael Z. (B. 7. LEXIN) <mzh...@bl...> - 2013-10-09 14:51:54
|
I try to write a test application in Microsoft VS 2010 as following.
and when I debug the code, it crash at:
writer.addDocument(&doc);
and I get an error message, it shows:
Runtime Error!
Program: C:...
R6025
- pure virtual function call
Anybody knows how to fix it?
Thanks.
___________________________________________________________________
RAMDirectory directory;
StandardAnalyzer analyzer;
IndexWriter writer(&directory, &analyzer, true);
lucene::document::Document doc;
doc.add(lucene::document::Field(L"content", L"abcd", lucene::document::Field::STORE_YES| lucene::document::Field::INDEX_TOKENIZED, true));
writer.addDocument(&doc);
writer.optimize();
writer.close();
|
|
From: Anthony N. <ant...@gm...> - 2013-08-29 07:26:13
|
Thanks Borek. I will take the latest from master for my testing. Anthony |
|
From: Kostka B. <ko...@to...> - 2013-08-19 10:04:43
|
I merged memleak_fixes into master.
Borek
> -----Original Message-----
> From: Kostka Bořivoj [mailto:ko...@to...]
> Sent: Monday, August 12, 2013 3:46 PM
> To: clu...@li...
> Subject: Re: [CLucene-dev] All remaining memory leaks in tests
> (hopefully) removed
>
> Hi Anthony,
>
> Thanks for the patch to DirectoryIndexReader. I incorporated it to
> memleak_fixes branch.
> I also merged bs2_memoryleak_fix branch provided by Veit into
> memleak_fixes branch.
>
> I'm going to merge everything into master soon (I want to run do some
> more tests before)
>
> Regards
>
> Borek
>
> > -----Original Message-----
> > From: Anthony Novatsis [mailto:ant...@gm...]
> > Sent: Saturday, August 10, 2013 12:56 AM
> > To: clu...@li...
> > Subject: Re: [CLucene-dev] All remaining memory leaks in tests
> > (hopefully) removed
> >
> > Hi Borek,
> >
> > Thank you very much for creating the memleak_fixes branch!
> >
> > We have actually seen some other memory leaks during testing.
> >
> > Firstly when DirectoryIndexReader::startCommit is called
> > consecutively, rollbackSegmentInfos is not freed if it was previously
> > allocated. The below patch fixes the leak and could be incorporated
> > into your branch if not already addressed (somebody with better
> > knowledge of the library may also want to review):
> >
> > --- a/clucene-core-
> > 2.3.3.4/src/core/CLucene/index/DirectoryIndexReader.cpp
> > +++ b/clucene-core-
> > 2.3.3.4/src/core/CLucene/index/DirectoryIndexReader.c
> > +++ pp
> > @@ -274,6 +274,9 @@ CL_NS_DEF(index)
> > */
> > void DirectoryIndexReader::startCommit() {
> > if (segmentInfos != NULL) {
> > + if (rollbackSegmentInfos != NULL) {
> > + _CLDELETE(rollbackSegmentInfos);
> > + }
> > rollbackSegmentInfos = segmentInfos->clone();
> > }
> > rollbackHasChanges = hasChanges;
> >
> > Secondly, we are seeing many leaks of
> > lucene::index::MergePolicy::OneMerge and lucene::index::SegmentInfos.
> > Unfortunately, I do not yet have a patch as I haven't been able to
> > investigate but will do so in the coming weeks.
> >
> > I am eager to see your branch merged into master to use that as the
> > basis for further memory leak investigation. Thanks again!
> >
> > Best regards,
> > Anthony
> >
> > ---------------------------------------------------------------------
> -
> > -
> > -------
> > Get 100% visibility into Java/.NET code with AppDynamics Lite!
> > It's a free troubleshooting tool designed for production.
> > Get down to code-level detail for bottlenecks, with <2% overhead.
> > Download for free and get started troubleshooting in minutes.
> >
> http://pubads.g.doubleclick.net/gampad/clk?id=48897031&iu=/4140/ostg.c
> > l
> > ktrk
> > _______________________________________________
> > CLucene-developers mailing list
> > CLu...@li...
> > https://lists.sourceforge.net/lists/listinfo/clucene-developers
> -----------------------------------------------------------------------
> -------
> Get 100% visibility into Java/.NET code with AppDynamics Lite!
> It's a free troubleshooting tool designed for production.
> Get down to code-level detail for bottlenecks, with <2% overhead.
> Download for free and get started troubleshooting in minutes.
> http://pubads.g.doubleclick.net/gampad/clk?id=48897031&iu=/4140/ostg.cl
> ktrk
> _______________________________________________
> CLucene-developers mailing list
> CLu...@li...
> https://lists.sourceforge.net/lists/listinfo/clucene-developers
|
|
From: Kostka B. <ko...@to...> - 2013-08-12 13:46:27
|
Hi Anthony,
Thanks for the patch to DirectoryIndexReader. I incorporated it to memleak_fixes branch.
I also merged bs2_memoryleak_fix branch provided by Veit into memleak_fixes branch.
I'm going to merge everything into master soon (I want to run do some more tests before)
Regards
Borek
> -----Original Message-----
> From: Anthony Novatsis [mailto:ant...@gm...]
> Sent: Saturday, August 10, 2013 12:56 AM
> To: clu...@li...
> Subject: Re: [CLucene-dev] All remaining memory leaks in tests
> (hopefully) removed
>
> Hi Borek,
>
> Thank you very much for creating the memleak_fixes branch!
>
> We have actually seen some other memory leaks during testing.
>
> Firstly when DirectoryIndexReader::startCommit is called consecutively,
> rollbackSegmentInfos is not freed if it was previously allocated. The
> below patch fixes the leak and could be incorporated into your branch
> if not already addressed (somebody with better knowledge of the library
> may also want to review):
>
> --- a/clucene-core-
> 2.3.3.4/src/core/CLucene/index/DirectoryIndexReader.cpp
> +++ b/clucene-core-
> 2.3.3.4/src/core/CLucene/index/DirectoryIndexReader.c
> +++ pp
> @@ -274,6 +274,9 @@ CL_NS_DEF(index)
> */
> void DirectoryIndexReader::startCommit() {
> if (segmentInfos != NULL) {
> + if (rollbackSegmentInfos != NULL) {
> + _CLDELETE(rollbackSegmentInfos);
> + }
> rollbackSegmentInfos = segmentInfos->clone();
> }
> rollbackHasChanges = hasChanges;
>
> Secondly, we are seeing many leaks of
> lucene::index::MergePolicy::OneMerge and lucene::index::SegmentInfos.
> Unfortunately, I do not yet have a patch as I haven't been able to
> investigate but will do so in the coming weeks.
>
> I am eager to see your branch merged into master to use that as the
> basis for further memory leak investigation. Thanks again!
>
> Best regards,
> Anthony
>
> -----------------------------------------------------------------------
> -------
> Get 100% visibility into Java/.NET code with AppDynamics Lite!
> It's a free troubleshooting tool designed for production.
> Get down to code-level detail for bottlenecks, with <2% overhead.
> Download for free and get started troubleshooting in minutes.
> http://pubads.g.doubleclick.net/gampad/clk?id=48897031&iu=/4140/ostg.cl
> ktrk
> _______________________________________________
> CLucene-developers mailing list
> CLu...@li...
> https://lists.sourceforge.net/lists/listinfo/clucene-developers
|
|
From: Anthony N. <ant...@gm...> - 2013-08-09 22:56:18
|
Hi Borek,
Thank you very much for creating the memleak_fixes branch!
We have actually seen some other memory leaks during testing.
Firstly when DirectoryIndexReader::startCommit is called
consecutively, rollbackSegmentInfos is not freed if it was previously
allocated. The below patch fixes the leak and could be incorporated
into your branch if not already addressed (somebody with better
knowledge of the library may also want to review):
--- a/clucene-core-2.3.3.4/src/core/CLucene/index/DirectoryIndexReader.cpp
+++ b/clucene-core-2.3.3.4/src/core/CLucene/index/DirectoryIndexReader.cpp
@@ -274,6 +274,9 @@ CL_NS_DEF(index)
*/
void DirectoryIndexReader::startCommit() {
if (segmentInfos != NULL) {
+ if (rollbackSegmentInfos != NULL) {
+ _CLDELETE(rollbackSegmentInfos);
+ }
rollbackSegmentInfos = segmentInfos->clone();
}
rollbackHasChanges = hasChanges;
Secondly, we are seeing many leaks of
lucene::index::MergePolicy::OneMerge and lucene::index::SegmentInfos.
Unfortunately, I do not yet have a patch as I haven't been able to
investigate but will do so in the coming weeks.
I am eager to see your branch merged into master to use that as the
basis for further memory leak investigation. Thanks again!
Best regards,
Anthony
|
|
From: Jonas P. <jon...@gm...> - 2013-08-04 12:25:00
|
Dear all
I have the following problem. When the application starts, the method
loadIndex is called and this method tries to load an existing index. This
method works fine when there is an index and when there is no index.
However during the execution of the program, the method startIndexing can
be called. When a new IndexWriter is created I get the following message:
Exception at 0x75ccc41f, code: 0xe06d7363: C++ exception, flags=0x1
(execution cannot be continued) (first chance) at
c:\users\clucene-core-2.3.3.4\clucene-core-2.3.3.4\src\core\clucene\store\directory.cpp:54
However when I check the index directory, I see that the index is being
generated by the method startIndexing. However when the execution flow
reaches the statement
IndexReader* r = IndexReader::open(indexDirectory.toStdString().c_str());
the following message is shown:
:0: error: Exception at 0x56e37bea, code: 0xc0000005: read access violation
at: 0x0, flags=0x0 (first chance)
I have no idea what the problem can be. I use Qt 5.1, Qt creator and MSVC
2010. I compiled the CLucene code as a DLL to which I link in the
application.
void IndexingEngine::loadIndex(Dataset *dataset, QString indexDirectory)
{
try{
if ( IndexReader::indexExists(indexDirectory.toStdString().c_str())
){
if ( IndexReader::isLocked(indexDirectory.toStdString().c_str())
){
IndexReader::unlock(indexDirectory.toStdString().c_str());
}
}
else
return;
QStringList *termsInIndex = new QStringList();
IndexReader* r = IndexReader
::open(indexDirectory.toStdString().c_str());
TermEnum* te = r->*terms*();
int32_t nterms;
for (nterms = 0; te->*next*() == true; nterms++) {
termsInIndex->append(QString::fromStdWString(te->*term*
()->text()));
}
_CLLDELETE(te);
r->close();
_CLLDELETE(r);
}catch(CLuceneError& err){
printf("Error: %s\n", err.what());
}catch(...){
printf("Unknown error\n");
}
}
void IndexingEngine::startIndexingTXT(Dataset *dataset, bool clearIndex,
QString indexDirectory)
{
int i=0;
const char* target = indexDirectory.toStdString().c_str();
IndexWriter* writer = NULL;
lucene::analysis::WhitespaceAnalyzer an;
try{
if ( !clearIndex && IndexReader::indexExists(target) ){
if ( IndexReader::isLocked(target) ){
IndexReader::unlock(target);
}
writer = _CLNEW IndexWriter( target, &an, false);
}else{
writer = _CLNEW IndexWriter(
indexDirectory.toStdString().c_str() ,&an, true);
}
writer->setMaxFieldLength(0x7FFFFFFFL); // LUCENE_INT32_MAX_SHOULDBE
writer->setUseCompoundFile(false);
{
QDirIterator it2 (dataset->getDatasetDirectory(), QDir::
NoDotAndDotDot | QDir::Files | QDir::Dirs);
while (it2.hasNext())
{
it2.next();
QFile file;
file.setFileName(it2.filePath());
file.*open*(QIODevice::ReadOnly | QIODevice::Text);
QString filename = it2.fileName();
QString contents = file.readAll();
Document doc;
doc.clear();
doc.add( *_CLNEW Field(_T("filename"),
filename.toStdWString().c_str(), Field::STORE_YES | Field::INDEX_UNTOKENIZED
) );
doc.add( *_CLNEW Field(_T("contents"),
contents.toStdWString().c_str(), Field::STORE_YES | Field::INDEX_TOKENIZED)
);
writer->addDocument( &doc );
i++;
file.*close*();
}
}
writer->setUseCompoundFile(true);
writer->optimize();
writer->close();
_CLLDELETE(writer);
QStringList *termsInIndex = new QStringList();
IndexReader* r = IndexReader
::open(indexDirectory.toStdString().c_str());
TermEnum* te = r->*terms*();
int32_t nterms;
for (nterms = 0; te->*next*() == true; nterms++) {
termsInIndex->append(QString::fromStdWString(te->*term*
()->text()));
}
_CLLDELETE(te);
r->close();
_CLLDELETE(r);
progress->setValue(100);
}catch(CLuceneError& err){
printf("Error: %s\n", err.what());
}catch(...){
printf("Unknown error\n");
}
_lucene_shutdown();
}
|
53 messages has been excluded from this view by a project administrator.
