Java Code Examples for org.apache.lucene.util.BytesRefBuilder#copyBytes()

The following examples show how to use org.apache.lucene.util.BytesRefBuilder#copyBytes() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FSTCompletionBuilder.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Builds the final automaton from a list of entries.
 */
private FST<Object> buildAutomaton(BytesRefSorter sorter) throws IOException {
  // Build the automaton.
  final Outputs<Object> outputs = NoOutputs.getSingleton();
  final Object empty = outputs.getNoOutput();
  final FSTCompiler<Object> fstCompiler = new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs)
      .shareMaxTailLength(shareMaxTailLength).build();

  BytesRefBuilder scratch = new BytesRefBuilder();
  BytesRef entry;
  final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();
  int count = 0;
  BytesRefIterator iter = sorter.iterator();
  while((entry = iter.next()) != null) {
    count++;
    if (scratch.get().compareTo(entry) != 0) {
      fstCompiler.add(Util.toIntsRef(entry, scratchIntsRef), empty);
      scratch.copyBytes(entry);
    }
  }
  
  return count == 0 ? null : fstCompiler.compile();
}
 
Example 2
Source File: TestLegacyNumericUtils.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testLongConversionAndOrdering() throws Exception {
  // generate a series of encoded longs, each numerical one bigger than the one before
  BytesRefBuilder last = new BytesRefBuilder();
  BytesRefBuilder act = new BytesRefBuilder();
  for (long l=-100000L; l<100000L; l++) {
    LegacyNumericUtils.longToPrefixCoded(l, 0, act);
    if (last!=null) {
      // test if smaller
      assertTrue("actual bigger than last (BytesRef)", last.get().compareTo(act.get()) < 0 );
      assertTrue("actual bigger than last (as String)", last.get().utf8ToString().compareTo(act.get().utf8ToString()) < 0 );
    }
    // test is back and forward conversion works
    assertEquals("forward and back conversion should generate same long", l, LegacyNumericUtils.prefixCodedToLong(act.get()));
    // next step
    last.copyBytes(act);
  }
}
 
Example 3
Source File: TestLegacyNumericUtils.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testIntConversionAndOrdering() throws Exception {
  // generate a series of encoded ints, each numerical one bigger than the one before
  BytesRefBuilder act = new BytesRefBuilder();
  BytesRefBuilder last = new BytesRefBuilder();
  for (int i=-100000; i<100000; i++) {
    LegacyNumericUtils.intToPrefixCoded(i, 0, act);
    if (last!=null) {
      // test if smaller
      assertTrue("actual bigger than last (BytesRef)", last.get().compareTo(act.get()) < 0 );
      assertTrue("actual bigger than last (as String)", last.get().utf8ToString().compareTo(act.get().utf8ToString()) < 0 );
    }
    // test is back and forward conversion works
    assertEquals("forward and back conversion should generate same int", i, LegacyNumericUtils.prefixCodedToInt(act.get()));
    // next step
    last.copyBytes(act.get());
  }
}
 
Example 4
Source File: DocTermsIndexDocValues.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public boolean bytesVal(int doc, BytesRefBuilder target) throws IOException {
  target.clear();
  if (getOrdForDoc(doc) == -1) {
    return false;
  } else {
    target.copyBytes(termsIndex.binaryValue());
    return true;
  }
}
 
Example 5
Source File: TestIndexWriterUnicode.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void checkTermsOrder(IndexReader r, Set<String> allTerms, boolean isTop) throws IOException {
  TermsEnum terms = MultiTerms.getTerms(r, "f").iterator();

  BytesRefBuilder last = new BytesRefBuilder();

  Set<String> seenTerms = new HashSet<>();

  while(true) {
    final BytesRef term = terms.next();
    if (term == null) {
      break;
    }

    assertTrue(last.get().compareTo(term) < 0);
    last.copyBytes(term);

    final String s = term.utf8ToString();
    assertTrue("term " + termDesc(s) + " was not added to index (count=" + allTerms.size() + ")", allTerms.contains(s));
    seenTerms.add(s);
  }

  if (isTop) {
    assertTrue(allTerms.equals(seenTerms));
  }

  // Test seeking:
  Iterator<String> it = seenTerms.iterator();
  while(it.hasNext()) {
    BytesRef tr = new BytesRef(it.next());
    assertEquals("seek failed for term=" + termDesc(tr.utf8ToString()),
                 TermsEnum.SeekStatus.FOUND,
                 terms.seekCeil(tr));
  }
}
 
Example 6
Source File: Uid.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
public static BytesRef createUidAsBytes(BytesRef type, BytesRef id, BytesRefBuilder spare) {
    spare.copyBytes(type);
    spare.append(DELIMITER_BYTES);
    spare.append(id);
    return spare.get();
}
 
Example 7
Source File: Lucene80DocValuesConsumer.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
private void writeTermsIndex(SortedSetDocValues values) throws IOException {
  final long size = values.getValueCount();
  meta.writeInt(Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT);
  long start = data.getFilePointer();

  long numBlocks = 1L + ((size + Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) >>> Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT);
  ByteBuffersDataOutput addressBuffer = new ByteBuffersDataOutput();
  DirectMonotonicWriter writer;
  try (ByteBuffersIndexOutput addressOutput = new ByteBuffersIndexOutput(addressBuffer, "temp", "temp")) {
    writer = DirectMonotonicWriter.getInstance(meta, addressOutput, numBlocks, DIRECT_MONOTONIC_BLOCK_SHIFT);
    TermsEnum iterator = values.termsEnum();
    BytesRefBuilder previous = new BytesRefBuilder();
    long offset = 0;
    long ord = 0;
    for (BytesRef term = iterator.next(); term != null; term = iterator.next()) {
      if ((ord & Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == 0) {
        writer.add(offset);
        final int sortKeyLength;
        if (ord == 0) {
          // no previous term: no bytes to write
          sortKeyLength = 0;
        } else {
          sortKeyLength = StringHelper.sortKeyLength(previous.get(), term);
        }
        offset += sortKeyLength;
        data.writeBytes(term.bytes, term.offset, sortKeyLength);
      } else if ((ord & Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) {
        previous.copyBytes(term);
      }
      ++ord;
    }
    writer.add(offset);
    writer.finish();
    meta.writeLong(start);
    meta.writeLong(data.getFilePointer() - start);
    start = data.getFilePointer();
    addressBuffer.copyTo(data);
    meta.writeLong(start);
    meta.writeLong(data.getFilePointer() - start);
  }
}
 
Example 8
Source File: TestDocumentsWriterDeleteQueue.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testStressDeleteQueue() throws Exception {
  DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(null);
  Set<Term> uniqueValues = new HashSet<>();
  final int size = 10000 + random().nextInt(500) * RANDOM_MULTIPLIER;
  Integer[] ids = new Integer[size];
  for (int i = 0; i < ids.length; i++) {
    ids[i] = random().nextInt();
    uniqueValues.add(new Term("id", ids[i].toString()));
  }
  CountDownLatch latch = new CountDownLatch(1);
  AtomicInteger index = new AtomicInteger(0);
  final int numThreads = 2 + random().nextInt(5);
  UpdateThread[] threads = new UpdateThread[numThreads];
  for (int i = 0; i < threads.length; i++) {
    threads[i] = new UpdateThread(queue, index, ids, latch);
    threads[i].start();
  }
  latch.countDown();
  for (int i = 0; i < threads.length; i++) {
    threads[i].join();
  }

  for (UpdateThread updateThread : threads) {
    DeleteSlice slice = updateThread.slice;
    queue.updateSlice(slice);
    BufferedUpdates deletes = updateThread.deletes;
    slice.apply(deletes, BufferedUpdates.MAX_INT);
    assertEquals(uniqueValues, deletes.deleteTerms.keySet());
  }
  queue.tryApplyGlobalSlice();
  Set<Term> frozenSet = new HashSet<>();
  BytesRefBuilder builder = new BytesRefBuilder();

  TermIterator iter = queue.freezeGlobalBuffer(null).deleteTerms.iterator();
  while (iter.next() != null) {
    builder.copyBytes(iter.bytes);
    frozenSet.add(new Term(iter.field(), builder.toBytesRef()));
  }

  assertEquals("num deletes must be 0 after freeze", 0, queue
      .numGlobalTermDeletes());
  assertEquals(uniqueValues.size(), frozenSet.size());
  assertEquals(uniqueValues, frozenSet);
}