如何在Lucene.Net 4.8中使用HIGH_COMPRESSION

juzqafwq 于 2022-11-07 发布在 Lucene

关注(0)|答案(1)|浏览(210)

我正在尝试尽可能压缩索引大小，需要帮助吗？https://lucenenet.apache.org/docs/4.8.0-beta00013/api/core/Lucene.Net.Codecs.Compressing.CompressionMode.html#Lucene_Net_Codecs_Compressing_CompressionMode_HIGH_COMPRESSION

public class LuceneIndexer
    {
        private Analyzer _analyzer = new ArabicAnalyzer(Lucene.Net.Util.LuceneVersion.LUCENE_48);
        private string _indexPath;
        private Directory _indexDirectory;
        public IndexWriter _indexWriter;

        public LuceneIndexer(string indexPath)
        {
            this._indexPath = indexPath;
            _indexDirectory = new SimpleFSDirectory(new System.IO.DirectoryInfo(_indexPath));
        }

        public void BuildCompleteIndex(IEnumerable<Document> documents)
        {
            IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Lucene.Net.Util.LuceneVersion.LUCENE_48, _analyzer) { OpenMode = OpenMode.CREATE_OR_APPEND };
            indexWriterConfig.MaxBufferedDocs = 2;
            indexWriterConfig.RAMBufferSizeMB = 128;
            indexWriterConfig.MaxThreadStates = 2;

            _indexWriter = new IndexWriter(_indexDirectory, indexWriterConfig);

            _indexWriter.AddDocuments(documents);

            _indexWriter.Flush(true, true);
            _indexWriter.Commit();

            _indexWriter.Dispose();
        }

        public IEnumerable<Document> Search(string searchTerm, string searchField, int limit)
        {
            IndexReader indexReader = DirectoryReader.Open(_indexDirectory);
            var searcher = new IndexSearcher(indexReader);
            var termQuery = new TermQuery(new Term(searchField, searchTerm)); // Lucene.Net.Util.LuceneVersion.LUCENE_48, searchField, _analyzer
            var hits = searcher.Search(termQuery, limit).ScoreDocs;

            var documents = new List<Document>();
            foreach (var hit in hits)
            {
                documents.Add(searcher.Doc(hit.Doc));
            }

            _analyzer.Dispose();
            return documents;
        }

    }

lucene

来源：https://stackoverflow.com/questions/65834288/how-to-use-high-compression-in-lucene-net-4-8

1条答案

按热度按时间

1tuwyuhd1#

首先要知道的是，Lucene索引有很多方面。当不使用复合文件时，这体现在创建的各种文件中。只看其中的两个，我们可以讨论称为发布的倒排索引，我们可以讨论存储的文档。在这两个中，据我所知，没有任何关于倒排索引压缩的可调设置。
HIGH_COMPRESSION模式与存储字段有关。如果您不存储字段，而只是使用Lucene.Net创建倒排索引，那么为存储字段启用高压缩并不会减少“Lucene索引”的大小。
也就是说，如果您 * 正在 * 存储字段，并希望对存储字段数据使用高压缩，那么您需要创建自己的编解码器，为存储字段启用高压缩。您首先需要一个启用了高压缩的Stored fields类。下面是这两个类，后面是一个单元测试，它使用了我为您编写的新编解码器。我还没有'我没有在大量数据上尝试过这段代码来查看效果，我把这段代码留给您作为练习，但这应该为使用High Compression对存储字段进行压缩指明了方向。

/*
     * Licensed to the Apache Software Foundation (ASF) under one or more
     * contributor license agreements.  See the NOTICE file distributed with
     * this work for additional information regarding copyright ownership.
     * The ASF licenses this file to You under the Apache License, Version 2.0
     * (the "License"); you may not use this file except in compliance with
     * the License.  You may obtain a copy of the License at
     *
     *     http://www.apache.org/licenses/LICENSE-2.0
     *
     * Unless required by applicable law or agreed to in writing, software
     * distributed under the License is distributed on an "AS IS" BASIS,
     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     * See the License for the specific language governing permissions and
     * limitations under the License.
     */

public sealed class Lucene41StoredFieldsHighCompressionFormat : CompressingStoredFieldsFormat {
        /// <summary>
        /// Sole constructor. </summary>
        public Lucene41StoredFieldsHighCompressionFormat()
            : base("Lucene41StoredFieldsHighCompression", CompressionMode.HIGH_COMPRESSION, 1 << 14) {
        }
    }

以下是使用此“高压缩”格式的自定义编解码器：

/*
     * Licensed to the Apache Software Foundation (ASF) under one or more
     * contributor license agreements.  See the NOTICE file distributed with
     * this work for additional information regarding copyright ownership.
     * The ASF licenses this file to You under the Apache License, Version 2.0
     * (the "License"); you may not use this file except in compliance with
     * the License.  You may obtain a copy of the License at
     *
     *     http://www.apache.org/licenses/LICENSE-2.0
     *
     * Unless required by applicable law or agreed to in writing, software
     * distributed under the License is distributed on an "AS IS" BASIS,
     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     * See the License for the specific language governing permissions and
     * limitations under the License.
     */

    using Lucene40LiveDocsFormat = Lucene.Net.Codecs.Lucene40.Lucene40LiveDocsFormat;
    using Lucene41StoredFieldsFormat = Lucene.Net.Codecs.Lucene41.Lucene41StoredFieldsFormat;
    using Lucene42NormsFormat = Lucene.Net.Codecs.Lucene42.Lucene42NormsFormat;
    using Lucene42TermVectorsFormat = Lucene.Net.Codecs.Lucene42.Lucene42TermVectorsFormat;
    using PerFieldDocValuesFormat = Lucene.Net.Codecs.PerField.PerFieldDocValuesFormat;
    using PerFieldPostingsFormat = Lucene.Net.Codecs.PerField.PerFieldPostingsFormat;

    /// <summary>
    /// Implements the Lucene 4.6 index format, with configurable per-field postings
    /// and docvalues formats.
    /// <para/>
    /// If you want to reuse functionality of this codec in another codec, extend
    /// <see cref="FilterCodec"/>.
    /// <para/>
    /// See <see cref="Lucene.Net.Codecs.Lucene46"/> package documentation for file format details.
    /// <para/>
    /// @lucene.experimental 
    /// </summary>
    // NOTE: if we make largish changes in a minor release, easier to just make Lucene46Codec or whatever
    // if they are backwards compatible or smallish we can probably do the backwards in the postingsreader
    // (it writes a minor version, etc).
    [CodecName("Lucene46HighCompression")]
    public class Lucene46HighCompressionCodec : Codec {
        private readonly StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsHighCompressionFormat();    //<--This is the only line different then the stock Lucene46Codec
        private readonly TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat();
        private readonly FieldInfosFormat fieldInfosFormat = new Lucene46FieldInfosFormat();
        private readonly SegmentInfoFormat segmentInfosFormat = new Lucene46SegmentInfoFormat();
        private readonly LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();

        private readonly PostingsFormat postingsFormat;

        private class PerFieldPostingsFormatAnonymousInnerClassHelper : PerFieldPostingsFormat {
            private readonly Lucene46HighCompressionCodec outerInstance;

            public PerFieldPostingsFormatAnonymousInnerClassHelper(Lucene46HighCompressionCodec outerInstance) {
                this.outerInstance = outerInstance;
            }

            [MethodImpl(MethodImplOptions.AggressiveInlining)]
            public override PostingsFormat GetPostingsFormatForField(string field) {
                return outerInstance.GetPostingsFormatForField(field);
            }
        }

        private readonly DocValuesFormat docValuesFormat;

        private class PerFieldDocValuesFormatAnonymousInnerClassHelper : PerFieldDocValuesFormat {
            private readonly Lucene46HighCompressionCodec outerInstance;

            public PerFieldDocValuesFormatAnonymousInnerClassHelper(Lucene46HighCompressionCodec outerInstance) {
                this.outerInstance = outerInstance;
            }

            [MethodImpl(MethodImplOptions.AggressiveInlining)]
            public override DocValuesFormat GetDocValuesFormatForField(string field) {
                return outerInstance.GetDocValuesFormatForField(field);
            }
        }

        /// <summary>
        /// Sole constructor. </summary>
        public Lucene46HighCompressionCodec()
            : base() {
            postingsFormat = new PerFieldPostingsFormatAnonymousInnerClassHelper(this);
            docValuesFormat = new PerFieldDocValuesFormatAnonymousInnerClassHelper(this);
        }

        public override sealed StoredFieldsFormat StoredFieldsFormat => fieldsFormat;

        public override sealed TermVectorsFormat TermVectorsFormat => vectorsFormat;

        public override sealed PostingsFormat PostingsFormat => postingsFormat;

        public override sealed FieldInfosFormat FieldInfosFormat => fieldInfosFormat;

        public override sealed SegmentInfoFormat SegmentInfoFormat => segmentInfosFormat;

        public override sealed LiveDocsFormat LiveDocsFormat => liveDocsFormat;

        /// <summary>
        /// Returns the postings format that should be used for writing
        /// new segments of <paramref name="field"/>.
        /// <para/>
        /// The default implementation always returns "Lucene41"
        /// </summary>
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public virtual PostingsFormat GetPostingsFormatForField(string field) {
            // LUCENENET specific - lazy initialize the codec to ensure we get the correct type if overridden.
            if (defaultFormat == null) {
                defaultFormat = Lucene.Net.Codecs.PostingsFormat.ForName("Lucene41");
            }
            return defaultFormat;
        }

        /// <summary>
        /// Returns the docvalues format that should be used for writing
        /// new segments of <paramref name="field"/>.
        /// <para/>
        /// The default implementation always returns "Lucene45"
        /// </summary>
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public virtual DocValuesFormat GetDocValuesFormatForField(string field) {
            // LUCENENET specific - lazy initialize the codec to ensure we get the correct type if overridden.
            if (defaultDVFormat == null) {
                defaultDVFormat = Lucene.Net.Codecs.DocValuesFormat.ForName("Lucene45");
            }
            return defaultDVFormat;
        }

        public override sealed DocValuesFormat DocValuesFormat => docValuesFormat;

        // LUCENENET specific - lazy initialize the codecs to ensure we get the correct type if overridden.
        private PostingsFormat defaultFormat;
        private DocValuesFormat defaultDVFormat;

        private readonly NormsFormat normsFormat = new Lucene42NormsFormat();

        public override sealed NormsFormat NormsFormat => normsFormat;
    }

感谢@NightOwl888，我现在明白，您还需要在启动时注册新的编解码器，如下所示：

Codec.SetCodecFactory(new DefaultCodecFactory {
    CustomCodecTypes = new Type[] { typeof(Lucene46HighCompressionCodec) }
});

以下是演示高压缩编解码器使用的单元测试：

public class TestCompression {

        [Fact]
        public void HighCompression() {
            FxTest.Setup();

            Directory indexDir = new RAMDirectory();

            Analyzer standardAnalyzer = new StandardAnalyzer(LuceneVersion.LUCENE_48);

            IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneVersion.LUCENE_48, standardAnalyzer);
            indexConfig.Codec = new Lucene46HighCompressionCodec();     //<--------Install the High Compression codec.

            indexConfig.UseCompoundFile = true;

            IndexWriter writer = new IndexWriter(indexDir, indexConfig);

            //souce: https://github.com/apache/lucenenet/blob/Lucene.Net_4_8_0_beta00006/src/Lucene.Net/Search/SearcherFactory.cs
            SearcherManager searcherManager = new SearcherManager(writer, applyAllDeletes: true, new SearchWarmer());

            Document doc = new Document();
            doc.Add(new StringField("examplePrimaryKey", "001", Field.Store.YES));
            doc.Add(new TextField("exampleField", "Unique gifts are great gifts.", Field.Store.YES));
            writer.AddDocument(doc);

            doc = new Document();
            doc.Add(new StringField("examplePrimaryKey", "002", Field.Store.YES));
            doc.Add(new TextField("exampleField", "Everyone is gifted.", Field.Store.YES));
            writer.AddDocument(doc);

            doc = new Document();
            doc.Add(new StringField("examplePrimaryKey", "003", Field.Store.YES));
            doc.Add(new TextField("exampleField", "Gifts are meant to be shared.", Field.Store.YES));
            writer.AddDocument(doc);

            writer.Commit();

            searcherManager.MaybeRefreshBlocking();
            IndexSearcher indexSearcher = searcherManager.Acquire();
            try {
                QueryParser parser = new QueryParser(LuceneVersion.LUCENE_48, "exampleField", standardAnalyzer);
                Query query = parser.Parse("everyone");

                TopDocs topDocs = indexSearcher.Search(query, int.MaxValue);

                int numMatchingDocs = topDocs.ScoreDocs.Length;
                Assert.Equal(1, numMatchingDocs);

                Document docRead = indexSearcher.Doc(topDocs.ScoreDocs[0].Doc);
                string primaryKey = docRead.Get("examplePrimaryKey");
                Assert.Equal("002", primaryKey);

            } finally {
                searcherManager.Release(indexSearcher);
            }

        }

    }

虽然我最初的回答是通过Lucene.Net github issue，但我在这里提供了答案，它将对Lucene .NET社区有更好的可见性，希望它也能帮助其他人。对于那些感兴趣的人，在该问题的线程的末尾有更多关于使用自定义编解码器的背景信息。

赞(0）回复(0）举报 2022-11-07

我来回答

如何在Lucene.Net 4.8中使用HIGH_COMPRESSION

1条答案

相关问题

热门标签

最新问答