org.apache.lucene.util.fst.Util.toIntsRef()方法的使用及代码示例

x33g5p2x  于2022-02-01 转载在 其他  
字(12.3k)|赞(0)|评价(0)|浏览(144)

本文整理了Java中org.apache.lucene.util.fst.Util.toIntsRef()方法的一些代码示例，展示了Util.toIntsRef()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Util.toIntsRef()方法的具体详情如下：
包路径：org.apache.lucene.util.fst.Util
类名称：Util
方法名：toIntsRef

Util.toIntsRef介绍

[英]Just takes unsigned byte values from the BytesRef and converts into an IntsRef.
[中]只需从字节引用中获取无符号字节值，并将其转换为IntsRef。

代码示例

代码示例来源：origin: org.apache.lucene/lucene-core

private void append(Builder<BytesRef> builder, FST<BytesRef> subIndex, IntsRefBuilder scratchIntsRef) throws IOException {
  final BytesRefFSTEnum<BytesRef> subIndexEnum = new BytesRefFSTEnum<>(subIndex);
  BytesRefFSTEnum.InputOutput<BytesRef> indexEnt;
  while((indexEnt = subIndexEnum.next()) != null) {
   //if (DEBUG) {
   //  System.out.println("      add sub=" + indexEnt.input + " " + indexEnt.input + " output=" + indexEnt.output);
   //}
   builder.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), indexEnt.output);
  }
 }
}

代码示例来源：origin: org.apache.lucene/lucene-core

assert bytes.length > 0;
scratchBytes.writeTo(bytes, 0);
indexBuilder.add(Util.toIntsRef(prefix, scratchIntsRef), new BytesRef(bytes, 0, bytes.length));
scratchBytes.reset();

代码示例来源：origin: org.elasticsearch/elasticsearch

public void finishTerm(long defaultWeight) throws IOException {
  ArrayUtil.timSort(surfaceFormsAndPayload, 0, count);
  int deduplicator = 0;
  analyzed.append((byte) 0);
  analyzed.setLength(analyzed.length() + 1);
  analyzed.grow(analyzed.length());
  for (int i = 0; i < count; i++) {
    analyzed.setByteAt(analyzed.length() - 1, (byte) deduplicator++);
    Util.toIntsRef(analyzed.get(), scratchInts);
    SurfaceFormAndPayload candiate = surfaceFormsAndPayload[i];
    long cost = candiate.weight == -1 ? encodeWeight(Math.min(Integer.MAX_VALUE, defaultWeight)) : candiate.weight;
    builder.add(scratchInts.get(), outputs.newPair(cost, candiate.payload));
  }
  seenSurfaceForms.clear();
  count = 0;
}

代码示例来源：origin: org.elasticsearch/elasticsearch

analyzed.append((byte) dedup);
Util.toIntsRef(analyzed.get(), scratchInts);

代码示例来源：origin: harbby/presto-connectors

/**
 * Adds a context with boost, set <code>exact</code> to false
 * if the context is a prefix of any indexed contexts
 */
public void addContext(CharSequence context, float boost, boolean exact) {
 if (boost < 0f) {
  throw new IllegalArgumentException("'boost' must be >= 0");
 }
 for (int i = 0; i < context.length(); i++) {
  if (ContextSuggestField.CONTEXT_SEPARATOR == context.charAt(i)) {
   throw new IllegalArgumentException("Illegal value [" + context + "] UTF-16 codepoint [0x"
     + Integer.toHexString((int) context.charAt(i))+ "] at position " + i + " is a reserved character");
  }
 }
 contexts.put(IntsRef.deepCopyOf(Util.toIntsRef(new BytesRef(context), scratch)), new ContextMetaData(boost, exact));
}

代码示例来源：origin: org.infinispan/infinispan-embedded-query

private void append(Builder<BytesRef> builder, FST<BytesRef> subIndex, IntsRefBuilder scratchIntsRef) throws IOException {
  final BytesRefFSTEnum<BytesRef> subIndexEnum = new BytesRefFSTEnum<>(subIndex);
  BytesRefFSTEnum.InputOutput<BytesRef> indexEnt;
  while((indexEnt = subIndexEnum.next()) != null) {
   //if (DEBUG) {
   //  System.out.println("      add sub=" + indexEnt.input + " " + indexEnt.input + " output=" + indexEnt.output);
   //}
   builder.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), indexEnt.output);
  }
 }
}

代码示例来源：origin: org.apache.lucene/lucene-sandbox

private void append(Builder<Pair<BytesRef,Long>> builder, FST<Pair<BytesRef,Long>> subIndex, IntsRefBuilder scratchIntsRef) throws IOException {
  final BytesRefFSTEnum<Pair<BytesRef,Long>> subIndexEnum = new BytesRefFSTEnum<>(subIndex);
  BytesRefFSTEnum.InputOutput<Pair<BytesRef,Long>> indexEnt;
  while((indexEnt = subIndexEnum.next()) != null) {
   //if (DEBUG) {
   //  System.out.println("      add sub=" + indexEnt.input + " " + indexEnt.input + " output=" + indexEnt.output);
   //}
   builder.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), indexEnt.output);
  }
 }
}

代码示例来源：origin: org.apache.servicemix.bundles/org.apache.servicemix.bundles.lucene

private void append(Builder<BytesRef> builder, FST<BytesRef> subIndex, IntsRefBuilder scratchIntsRef) throws IOException {
  final BytesRefFSTEnum<BytesRef> subIndexEnum = new BytesRefFSTEnum<>(subIndex);
  BytesRefFSTEnum.InputOutput<BytesRef> indexEnt;
  while((indexEnt = subIndexEnum.next()) != null) {
   //if (DEBUG) {
   //  System.out.println("      add sub=" + indexEnt.input + " " + indexEnt.input + " output=" + indexEnt.output);
   //}
   builder.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), indexEnt.output);
  }
 }
}

代码示例来源：origin: harbby/presto-connectors

private void append(Builder<BytesRef> builder, FST<BytesRef> subIndex, IntsRefBuilder scratchIntsRef) throws IOException {
  final BytesRefFSTEnum<BytesRef> subIndexEnum = new BytesRefFSTEnum<>(subIndex);
  BytesRefFSTEnum.InputOutput<BytesRef> indexEnt;
  while((indexEnt = subIndexEnum.next()) != null) {
   //if (DEBUG) {
   //  System.out.println("      add sub=" + indexEnt.input + " " + indexEnt.input + " output=" + indexEnt.output);
   //}
   builder.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), indexEnt.output);
  }
 }
}

代码示例来源：origin: harbby/presto-connectors

private void append(Builder<Pair<BytesRef,Long>> builder, FST<Pair<BytesRef,Long>> subIndex, IntsRefBuilder scratchIntsRef) throws IOException {
  final BytesRefFSTEnum<Pair<BytesRef,Long>> subIndexEnum = new BytesRefFSTEnum<>(subIndex);
  BytesRefFSTEnum.InputOutput<Pair<BytesRef,Long>> indexEnt;
  while((indexEnt = subIndexEnum.next()) != null) {
   //if (DEBUG) {
   //  System.out.println("      add sub=" + indexEnt.input + " " + indexEnt.input + " output=" + indexEnt.output);
   //}
   builder.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), indexEnt.output);
  }
 }
}

代码示例来源：origin: org.apache.lucene/lucene-classification

private void updateFST(SortedMap<String, Double> weights) throws IOException {
 PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
 Builder<Long> fstBuilder = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs);
 BytesRefBuilder scratchBytes = new BytesRefBuilder();
 IntsRefBuilder scratchInts = new IntsRefBuilder();
 for (Map.Entry<String, Double> entry : weights.entrySet()) {
  scratchBytes.copyChars(entry.getKey());
  fstBuilder.add(Util.toIntsRef(scratchBytes.get(), scratchInts), entry
      .getValue().longValue());
 }
 fst = fstBuilder.finish();
}

代码示例来源：origin: org.apache.lucene/lucene-codecs

private void append(Builder<Output> builder, FST<Output> subIndex, long termOrdOffset, IntsRefBuilder scratchIntsRef) throws IOException {
  final BytesRefFSTEnum<Output> subIndexEnum = new BytesRefFSTEnum<>(subIndex);
  BytesRefFSTEnum.InputOutput<Output> indexEnt;
  while ((indexEnt = subIndexEnum.next()) != null) {
   //if (DEBUG) {
   //  System.out.println("      add sub=" + indexEnt.input + " " + indexEnt.input + " output=" + indexEnt.output);
   //}
   Output output = indexEnt.output;
   //long blockTermCount = output.endOrd - output.startOrd + 1;
   Output newOutput = FST_OUTPUTS.newOutput(output.bytes, termOrdOffset+output.startOrd, output.endOrd-termOrdOffset);
   //System.out.println("  append sub=" + indexEnt.input + " output=" + indexEnt.output + " termOrdOffset=" + termOrdOffset + " blockTermCount=" + blockTermCount  + " newOutput=" + newOutput  + " endOrd=" + (termOrdOffset+Long.MAX_VALUE-output.endOrd));
   builder.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), newOutput);
  }
 }
}

代码示例来源：origin: org.apache.lucene/lucene-codecs

@Override
public void add(BytesRef text, TermStats stats, long termsFilePointer) throws IOException {
 if (text.length == 0) {
  // We already added empty string in ctor
  assert termsFilePointer == startTermsFilePointer;
  return;
 }
 final int lengthSave = text.length;
 text.length = indexedTermPrefixLength(lastTerm.get(), text);
 try {
  fstBuilder.add(Util.toIntsRef(text, scratchIntsRef), termsFilePointer);
 } finally {
  text.length = lengthSave;
 }
 lastTerm.copyBytes(text);
}

代码示例来源：origin: org.apache.lucene/lucene-codecs

public void finishTerm(BytesRef text, BlockTermState state) throws IOException {
 // write term meta data into fst
 final FSTTermOutputs.TermData meta = new FSTTermOutputs.TermData();
 meta.longs = new long[longsSize];
 meta.bytes = null;
 meta.docFreq = state.docFreq;
 meta.totalTermFreq = state.totalTermFreq;
 postingsWriter.encodeTerm(meta.longs, metaWriter, fieldInfo, state, true);
 final int bytesSize = (int)metaWriter.getFilePointer();
 if (bytesSize > 0) {
  meta.bytes = new byte[bytesSize];
  metaWriter.writeTo(meta.bytes, 0);
  metaWriter.reset();
 }
 builder.add(Util.toIntsRef(text, scratchTerm), meta);
 numTerms++;
}

代码示例来源：origin: com.strapdata.elasticsearch/elasticsearch

public void finishTerm(long defaultWeight) throws IOException {
  ArrayUtil.timSort(surfaceFormsAndPayload, 0, count);
  int deduplicator = 0;
  analyzed.append((byte) 0);
  analyzed.setLength(analyzed.length() + 1);
  analyzed.grow(analyzed.length());
  for (int i = 0; i < count; i++) {
    analyzed.setByteAt(analyzed.length() - 1, (byte) deduplicator++);
    Util.toIntsRef(analyzed.get(), scratchInts);
    SurfaceFormAndPayload candiate = surfaceFormsAndPayload[i];
    long cost = candiate.weight == -1 ? encodeWeight(Math.min(Integer.MAX_VALUE, defaultWeight)) : candiate.weight;
    builder.add(scratchInts.get(), outputs.newPair(cost, candiate.payload));
  }
  seenSurfaceForms.clear();
  count = 0;
}

代码示例来源：origin: org.apache.servicemix.bundles/org.apache.servicemix.bundles.elasticsearch

public void finishTerm(long defaultWeight) throws IOException {
  ArrayUtil.timSort(surfaceFormsAndPayload, 0, count);
  int deduplicator = 0;
  analyzed.append((byte) 0);
  analyzed.setLength(analyzed.length() + 1);
  analyzed.grow(analyzed.length());
  for (int i = 0; i < count; i++) {
    analyzed.setByteAt(analyzed.length() - 1, (byte) deduplicator++);
    Util.toIntsRef(analyzed.get(), scratchInts);
    SurfaceFormAndPayload candiate = surfaceFormsAndPayload[i];
    long cost = candiate.weight == -1 ? encodeWeight(Math.min(Integer.MAX_VALUE, defaultWeight)) : candiate.weight;
    builder.add(scratchInts.get(), outputs.newPair(cost, candiate.payload));
  }
  seenSurfaceForms.clear();
  count = 0;
}

代码示例来源：origin: apache/servicemix-bundles

public void finishTerm(long defaultWeight) throws IOException {
  ArrayUtil.timSort(surfaceFormsAndPayload, 0, count);
  int deduplicator = 0;
  analyzed.append((byte) 0);
  analyzed.setLength(analyzed.length() + 1);
  analyzed.grow(analyzed.length());
  for (int i = 0; i < count; i++) {
    analyzed.setByteAt(analyzed.length() - 1, (byte) deduplicator++);
    Util.toIntsRef(analyzed.get(), scratchInts);
    SurfaceFormAndPayload candiate = surfaceFormsAndPayload[i];
    long cost = candiate.weight == -1 ? encodeWeight(Math.min(Integer.MAX_VALUE, defaultWeight)) : candiate.weight;
    builder.add(scratchInts.get(), outputs.newPair(cost, candiate.payload));
  }
  seenSurfaceForms.clear();
  count = 0;
}

代码示例来源：origin: harbby/presto-connectors

public void finishTerm(long defaultWeight) throws IOException {
  ArrayUtil.timSort(surfaceFormsAndPayload, 0, count);
  int deduplicator = 0;
  analyzed.append((byte) 0);
  analyzed.setLength(analyzed.length() + 1);
  analyzed.grow(analyzed.length());
  for (int i = 0; i < count; i++) {
    analyzed.setByteAt(analyzed.length() - 1, (byte) deduplicator++);
    Util.toIntsRef(analyzed.get(), scratchInts);
    SurfaceFormAndPayload candiate = surfaceFormsAndPayload[i];
    long cost = candiate.weight == -1 ? encodeWeight(Math.min(Integer.MAX_VALUE, defaultWeight)) : candiate.weight;
    builder.add(scratchInts.get(), outputs.newPair(cost, candiate.payload));
  }
  seenSurfaceForms.clear();
  count = 0;
}

代码示例来源：origin: harbby/presto-connectors

/**
 * Writes all the entries for the FST input term
 */
public void finishTerm() throws IOException {
 int numArcs = 0;
 int numDedupBytes = 1;
 analyzed.grow(analyzed.length() + 1);
 analyzed.setLength(analyzed.length() + 1);
 for (Entry entry : entries) {
  if (numArcs == maxNumArcsForDedupByte(numDedupBytes)) {
   analyzed.setByteAt(analyzed.length() - 1, (byte) (numArcs));
   analyzed.grow(analyzed.length() + 1);
   analyzed.setLength(analyzed.length() + 1);
   numArcs = 0;
   numDedupBytes++;
  }
  analyzed.setByteAt(analyzed.length() - 1, (byte) numArcs++);
  Util.toIntsRef(analyzed.get(), scratchInts);
  builder.add(scratchInts.get(), outputs.newPair(entry.weight, entry.payload));
 }
 maxAnalyzedPathsPerOutput = Math.max(maxAnalyzedPathsPerOutput, entries.size());
 entries.clear();
}

代码示例来源：origin: org.apache.lucene/lucene-codecs

private void writeFST(FieldInfo field, Iterable<BytesRef> values) throws IOException {
 meta.writeVInt(field.number);
 meta.writeByte(FST);
 meta.writeLong(data.getFilePointer());
 PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
 Builder<Long> builder = new Builder<>(INPUT_TYPE.BYTE1, outputs);
 IntsRefBuilder scratch = new IntsRefBuilder();
 long ord = 0;
 for (BytesRef v : values) {
  builder.add(Util.toIntsRef(v, scratch), ord);
  ord++;
 }
 FST<Long> fst = builder.finish();
 if (fst != null) {
  fst.save(data);
 }
 meta.writeVLong(ord);
}

内容来源于网络，如有侵权，请联系作者删除！

Util

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备2022004421号-2 NULL123 https://www.null123.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com