Java Code Examples for org.apache.lucene.util.BytesRefBuilder#copyBytes()

/**
 * Builds the final automaton from a list of entries.
 */
private FST<Object> buildAutomaton(BytesRefSorter sorter) throws IOException {
  // Build the automaton.
  final Outputs<Object> outputs = NoOutputs.getSingleton();
  final Object empty = outputs.getNoOutput();
  final FSTCompiler<Object> fstCompiler = new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs)
      .shareMaxTailLength(shareMaxTailLength).build();

  BytesRefBuilder scratch = new BytesRefBuilder();
  BytesRef entry;
  final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();
  int count = 0;
  BytesRefIterator iter = sorter.iterator();
  while((entry = iter.next()) != null) {
    count++;
    if (scratch.get().compareTo(entry) != 0) {
      fstCompiler.add(Util.toIntsRef(entry, scratchIntsRef), empty);
      scratch.copyBytes(entry);
    }
  }
  
  return count == 0 ? null : fstCompiler.compile();
}
 

public void testLongConversionAndOrdering() throws Exception {
  // generate a series of encoded longs, each numerical one bigger than the one before
  BytesRefBuilder last = new BytesRefBuilder();
  BytesRefBuilder act = new BytesRefBuilder();
  for (long l=-100000L; l<100000L; l++) {
    LegacyNumericUtils.longToPrefixCoded(l, 0, act);
    if (last!=null) {
      // test if smaller
      assertTrue("actual bigger than last (BytesRef)", last.get().compareTo(act.get()) < 0 );
      assertTrue("actual bigger than last (as String)", last.get().utf8ToString().compareTo(act.get().utf8ToString()) < 0 );
    }
    // test is back and forward conversion works
    assertEquals("forward and back conversion should generate same long", l, LegacyNumericUtils.prefixCodedToLong(act.get()));
    // next step
    last.copyBytes(act);
  }
}
 

public void testIntConversionAndOrdering() throws Exception {
  // generate a series of encoded ints, each numerical one bigger than the one before
  BytesRefBuilder act = new BytesRefBuilder();
  BytesRefBuilder last = new BytesRefBuilder();
  for (int i=-100000; i<100000; i++) {
    LegacyNumericUtils.intToPrefixCoded(i, 0, act);
    if (last!=null) {
      // test if smaller
      assertTrue("actual bigger than last (BytesRef)", last.get().compareTo(act.get()) < 0 );
      assertTrue("actual bigger than last (as String)", last.get().utf8ToString().compareTo(act.get().utf8ToString()) < 0 );
    }
    // test is back and forward conversion works
    assertEquals("forward and back conversion should generate same int", i, LegacyNumericUtils.prefixCodedToInt(act.get()));
    // next step
    last.copyBytes(act.get());
  }
}
 

@Override
public boolean bytesVal(int doc, BytesRefBuilder target) throws IOException {
  target.clear();
  if (getOrdForDoc(doc) == -1) {
    return false;
  } else {
    target.copyBytes(termsIndex.binaryValue());
    return true;
  }
}
 

private void checkTermsOrder(IndexReader r, Set<String> allTerms, boolean isTop) throws IOException {
  TermsEnum terms = MultiTerms.getTerms(r, "f").iterator();

  BytesRefBuilder last = new BytesRefBuilder();

  Set<String> seenTerms = new HashSet<>();

  while(true) {
    final BytesRef term = terms.next();
    if (term == null) {
      break;
    }

    assertTrue(last.get().compareTo(term) < 0);
    last.copyBytes(term);

    final String s = term.utf8ToString();
    assertTrue("term " + termDesc(s) + " was not added to index (count=" + allTerms.size() + ")", allTerms.contains(s));
    seenTerms.add(s);
  }

  if (isTop) {
    assertTrue(allTerms.equals(seenTerms));
  }

  // Test seeking:
  Iterator<String> it = seenTerms.iterator();
  while(it.hasNext()) {
    BytesRef tr = new BytesRef(it.next());
    assertEquals("seek failed for term=" + termDesc(tr.utf8ToString()),
                 TermsEnum.SeekStatus.FOUND,
                 terms.seekCeil(tr));
  }
}
 

public static BytesRef createUidAsBytes(BytesRef type, BytesRef id, BytesRefBuilder spare) {
    spare.copyBytes(type);
    spare.append(DELIMITER_BYTES);
    spare.append(id);
    return spare.get();
}
 

private void writeTermsIndex(SortedSetDocValues values) throws IOException {
  final long size = values.getValueCount();
  meta.writeInt(Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT);
  long start = data.getFilePointer();

  long numBlocks = 1L + ((size + Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) >>> Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT);
  ByteBuffersDataOutput addressBuffer = new ByteBuffersDataOutput();
  DirectMonotonicWriter writer;
  try (ByteBuffersIndexOutput addressOutput = new ByteBuffersIndexOutput(addressBuffer, "temp", "temp")) {
    writer = DirectMonotonicWriter.getInstance(meta, addressOutput, numBlocks, DIRECT_MONOTONIC_BLOCK_SHIFT);
    TermsEnum iterator = values.termsEnum();
    BytesRefBuilder previous = new BytesRefBuilder();
    long offset = 0;
    long ord = 0;
    for (BytesRef term = iterator.next(); term != null; term = iterator.next()) {
      if ((ord & Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == 0) {
        writer.add(offset);
        final int sortKeyLength;
        if (ord == 0) {
          // no previous term: no bytes to write
          sortKeyLength = 0;
        } else {
          sortKeyLength = StringHelper.sortKeyLength(previous.get(), term);
        }
        offset += sortKeyLength;
        data.writeBytes(term.bytes, term.offset, sortKeyLength);
      } else if ((ord & Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) {
        previous.copyBytes(term);
      }
      ++ord;
    }
    writer.add(offset);
    writer.finish();
    meta.writeLong(start);
    meta.writeLong(data.getFilePointer() - start);
    start = data.getFilePointer();
    addressBuffer.copyTo(data);
    meta.writeLong(start);
    meta.writeLong(data.getFilePointer() - start);
  }
}
 

public void testStressDeleteQueue() throws Exception {
  DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(null);
  Set<Term> uniqueValues = new HashSet<>();
  final int size = 10000 + random().nextInt(500) * RANDOM_MULTIPLIER;
  Integer[] ids = new Integer[size];
  for (int i = 0; i < ids.length; i++) {
    ids[i] = random().nextInt();
    uniqueValues.add(new Term("id", ids[i].toString()));
  }
  CountDownLatch latch = new CountDownLatch(1);
  AtomicInteger index = new AtomicInteger(0);
  final int numThreads = 2 + random().nextInt(5);
  UpdateThread[] threads = new UpdateThread[numThreads];
  for (int i = 0; i < threads.length; i++) {
    threads[i] = new UpdateThread(queue, index, ids, latch);
    threads[i].start();
  }
  latch.countDown();
  for (int i = 0; i < threads.length; i++) {
    threads[i].join();
  }

  for (UpdateThread updateThread : threads) {
    DeleteSlice slice = updateThread.slice;
    queue.updateSlice(slice);
    BufferedUpdates deletes = updateThread.deletes;
    slice.apply(deletes, BufferedUpdates.MAX_INT);
    assertEquals(uniqueValues, deletes.deleteTerms.keySet());
  }
  queue.tryApplyGlobalSlice();
  Set<Term> frozenSet = new HashSet<>();
  BytesRefBuilder builder = new BytesRefBuilder();

  TermIterator iter = queue.freezeGlobalBuffer(null).deleteTerms.iterator();
  while (iter.next() != null) {
    builder.copyBytes(iter.bytes);
    frozenSet.add(new Term(iter.field(), builder.toBytesRef()));
  }

  assertEquals("num deletes must be 0 after freeze", 0, queue
      .numGlobalTermDeletes());
  assertEquals(uniqueValues.size(), frozenSet.size());
  assertEquals(uniqueValues, frozenSet);
}