#2447 UTF8 encoding is longer than the max length 32766

[Anonymous]

program.name=DocFetcher
program.version=1.1.27
program.build=20260119-235832
program.portable=false
java.runtime.name=OpenJDK Runtime Environment
java.runtime.version=21.0.8+9-LTS
java.version=21.0.8
sun.arch.data.model=64
os.arch=amd64
os.name=Windows 10
os.version=10.0
user.language=en
java.lang.IllegalArgumentException: Document contains at least one immense term in field="uid" (whose UTF8 encoding is longer than the max length 32766), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '[102, 105, 108, 101, 58, 47, 47, 67, 58, 47, 80, 114, 111, 103, 114, 97, 109, 32, 70, 105, 108, 101, 115, 47, 71, 111, 47, 115, 114, 99]...', original message: bytes can be at most 32766 in length; got 1048641
at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:806)
at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:447)
at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:403)
at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:232)
at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:478)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1571)
at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1316)
at net.sourceforge.docfetcher.model.index.IndexWriterAdapter.add(IndexWriterAdapter.java:83)
at net.sourceforge.docfetcher.model.index.file.SimpleDocWriter.write(SimpleDocWriter.java:44)
at net.sourceforge.docfetcher.model.index.file.LuceneDocWriter.add(LuceneDocWriter.java:36)
at net.sourceforge.docfetcher.model.index.file.FileContext.index(FileContext.java:161)
at net.sourceforge.docfetcher.model.index.file.FileIndex$1.handleFile(FileIndex.java:295)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.runWithHtmlPairing(HtmlFileLister.java:126)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.doRun(HtmlFileLister.java:57)
at net.sourceforge.docfetcher.util.Stoppable.run(Stoppable.java:57)
at net.sourceforge.docfetcher.model.index.file.FileIndex.visitDirOrZip(FileIndex.java:442)
at net.sourceforge.docfetcher.model.index.file.FileIndex$1.handleDir(FileIndex.java:393)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.runWithHtmlPairing(HtmlFileLister.java:147)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.doRun(HtmlFileLister.java:57)
at net.sourceforge.docfetcher.util.Stoppable.run(Stoppable.java:57)
at net.sourceforge.docfetcher.model.index.file.FileIndex.visitDirOrZip(FileIndex.java:442)
at net.sourceforge.docfetcher.model.index.file.FileIndex$1.handleDir(FileIndex.java:393)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.runWithHtmlPairing(HtmlFileLister.java:147)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.doRun(HtmlFileLister.java:57)
at net.sourceforge.docfetcher.util.Stoppable.run(Stoppable.java:57)
at net.sourceforge.docfetcher.model.index.file.FileIndex.visitDirOrZip(FileIndex.java:442)
at net.sourceforge.docfetcher.model.index.file.FileIndex$1.handleDir(FileIndex.java:393)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.runWithHtmlPairing(HtmlFileLister.java:147)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.doRun(HtmlFileLister.java:57)
at net.sourceforge.docfetcher.util.Stoppable.run(Stoppable.java:57)
at net.sourceforge.docfetcher.model.index.file.FileIndex.visitDirOrZip(FileIndex.java:442)
at net.sourceforge.docfetcher.model.index.file.FileIndex$1.handleDir(FileIndex.java:393)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.runWithHtmlPairing(HtmlFileLister.java:147)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.doRun(HtmlFileLister.java:57)
at net.sourceforge.docfetcher.util.Stoppable.run(Stoppable.java:57)
at net.sourceforge.docfetcher.model.index.file.FileIndex.visitDirOrZip(FileIndex.java:442)
at net.sourceforge.docfetcher.model.index.file.FileIndex$1.handleDir(FileIndex.java:393)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.runWithHtmlPairing(HtmlFileLister.java:147)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.doRun(HtmlFileLister.java:57)
at net.sourceforge.docfetcher.util.Stoppable.run(Stoppable.java:57)
at net.sourceforge.docfetcher.model.index.file.FileIndex.visitDirOrZip(FileIndex.java:442)
at net.sourceforge.docfetcher.model.index.file.FileIndex$1.handleDir(FileIndex.java:393)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.runWithHtmlPairing(HtmlFileLister.java:147)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.doRun(HtmlFileLister.java:57)
at net.sourceforge.docfetcher.util.Stoppable.run(Stoppable.java:57)
at net.sourceforge.docfetcher.model.index.file.FileIndex.visitDirOrZip(FileIndex.java:442)
at net.sourceforge.docfetcher.model.index.file.FileIndex$1.handleDir(FileIndex.java:393)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.runWithHtmlPairing(HtmlFileLister.java:147)
at net.sourceforge.docfetcher.model.index.file.HtmlFileLister.doRun(HtmlFileLister.java:57)
at net.sourceforge.docfetcher.util.Stoppable.run(Stoppable.java:57)
at net.sourceforge.docfetcher.model.index.file.FileIndex.visitDirOrZip(FileIndex.java:442)
at net.sourceforge.docfetcher.model.index.file.FileIndex.doUpdate(FileIndex.java:159)
at net.sourceforge.docfetcher.model.TreeIndex.update(TreeIndex.java:149)
at net.sourceforge.docfetcher.model.index.Task.update(Task.java:98)
at net.sourceforge.docfetcher.model.index.IndexingQueue.threadLoop(IndexingQueue.java:193)
at net.sourceforge.docfetcher.model.index.IndexingQueue$2.run(IndexingQueue.java:118)
Caused by: org.apache.lucene.util.BytesRefHash$MaxBytesLengthExceededException: bytes can be at most 32766 in length; got 1048641
at org.apache.lucene.util.BytesRefHash.add(BytesRefHash.java:263)
at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:149)
at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:796)
... 55 more

[Nam-Quang Tran]

Will be fixed in DocFetcher 1.1.28.