Java Code Examples for org.apache.lucene.util.BytesRefBuilder#copyBytes()
The following examples show how to use
org.apache.lucene.util.BytesRefBuilder#copyBytes() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FSTCompletionBuilder.java From lucene-solr with Apache License 2.0 | 6 votes |
/** * Builds the final automaton from a list of entries. */ private FST<Object> buildAutomaton(BytesRefSorter sorter) throws IOException { // Build the automaton. final Outputs<Object> outputs = NoOutputs.getSingleton(); final Object empty = outputs.getNoOutput(); final FSTCompiler<Object> fstCompiler = new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs) .shareMaxTailLength(shareMaxTailLength).build(); BytesRefBuilder scratch = new BytesRefBuilder(); BytesRef entry; final IntsRefBuilder scratchIntsRef = new IntsRefBuilder(); int count = 0; BytesRefIterator iter = sorter.iterator(); while((entry = iter.next()) != null) { count++; if (scratch.get().compareTo(entry) != 0) { fstCompiler.add(Util.toIntsRef(entry, scratchIntsRef), empty); scratch.copyBytes(entry); } } return count == 0 ? null : fstCompiler.compile(); }
Example 2
Source File: TestLegacyNumericUtils.java From lucene-solr with Apache License 2.0 | 6 votes |
public void testLongConversionAndOrdering() throws Exception { // generate a series of encoded longs, each numerical one bigger than the one before BytesRefBuilder last = new BytesRefBuilder(); BytesRefBuilder act = new BytesRefBuilder(); for (long l=-100000L; l<100000L; l++) { LegacyNumericUtils.longToPrefixCoded(l, 0, act); if (last!=null) { // test if smaller assertTrue("actual bigger than last (BytesRef)", last.get().compareTo(act.get()) < 0 ); assertTrue("actual bigger than last (as String)", last.get().utf8ToString().compareTo(act.get().utf8ToString()) < 0 ); } // test is back and forward conversion works assertEquals("forward and back conversion should generate same long", l, LegacyNumericUtils.prefixCodedToLong(act.get())); // next step last.copyBytes(act); } }
Example 3
Source File: TestLegacyNumericUtils.java From lucene-solr with Apache License 2.0 | 6 votes |
public void testIntConversionAndOrdering() throws Exception { // generate a series of encoded ints, each numerical one bigger than the one before BytesRefBuilder act = new BytesRefBuilder(); BytesRefBuilder last = new BytesRefBuilder(); for (int i=-100000; i<100000; i++) { LegacyNumericUtils.intToPrefixCoded(i, 0, act); if (last!=null) { // test if smaller assertTrue("actual bigger than last (BytesRef)", last.get().compareTo(act.get()) < 0 ); assertTrue("actual bigger than last (as String)", last.get().utf8ToString().compareTo(act.get().utf8ToString()) < 0 ); } // test is back and forward conversion works assertEquals("forward and back conversion should generate same int", i, LegacyNumericUtils.prefixCodedToInt(act.get())); // next step last.copyBytes(act.get()); } }
Example 4
Source File: DocTermsIndexDocValues.java From lucene-solr with Apache License 2.0 | 5 votes |
@Override public boolean bytesVal(int doc, BytesRefBuilder target) throws IOException { target.clear(); if (getOrdForDoc(doc) == -1) { return false; } else { target.copyBytes(termsIndex.binaryValue()); return true; } }
Example 5
Source File: TestIndexWriterUnicode.java From lucene-solr with Apache License 2.0 | 5 votes |
private void checkTermsOrder(IndexReader r, Set<String> allTerms, boolean isTop) throws IOException { TermsEnum terms = MultiTerms.getTerms(r, "f").iterator(); BytesRefBuilder last = new BytesRefBuilder(); Set<String> seenTerms = new HashSet<>(); while(true) { final BytesRef term = terms.next(); if (term == null) { break; } assertTrue(last.get().compareTo(term) < 0); last.copyBytes(term); final String s = term.utf8ToString(); assertTrue("term " + termDesc(s) + " was not added to index (count=" + allTerms.size() + ")", allTerms.contains(s)); seenTerms.add(s); } if (isTop) { assertTrue(allTerms.equals(seenTerms)); } // Test seeking: Iterator<String> it = seenTerms.iterator(); while(it.hasNext()) { BytesRef tr = new BytesRef(it.next()); assertEquals("seek failed for term=" + termDesc(tr.utf8ToString()), TermsEnum.SeekStatus.FOUND, terms.seekCeil(tr)); } }
Example 6
Source File: Uid.java From Elasticsearch with Apache License 2.0 | 4 votes |
public static BytesRef createUidAsBytes(BytesRef type, BytesRef id, BytesRefBuilder spare) { spare.copyBytes(type); spare.append(DELIMITER_BYTES); spare.append(id); return spare.get(); }
Example 7
Source File: Lucene80DocValuesConsumer.java From lucene-solr with Apache License 2.0 | 4 votes |
private void writeTermsIndex(SortedSetDocValues values) throws IOException { final long size = values.getValueCount(); meta.writeInt(Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT); long start = data.getFilePointer(); long numBlocks = 1L + ((size + Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) >>> Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT); ByteBuffersDataOutput addressBuffer = new ByteBuffersDataOutput(); DirectMonotonicWriter writer; try (ByteBuffersIndexOutput addressOutput = new ByteBuffersIndexOutput(addressBuffer, "temp", "temp")) { writer = DirectMonotonicWriter.getInstance(meta, addressOutput, numBlocks, DIRECT_MONOTONIC_BLOCK_SHIFT); TermsEnum iterator = values.termsEnum(); BytesRefBuilder previous = new BytesRefBuilder(); long offset = 0; long ord = 0; for (BytesRef term = iterator.next(); term != null; term = iterator.next()) { if ((ord & Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == 0) { writer.add(offset); final int sortKeyLength; if (ord == 0) { // no previous term: no bytes to write sortKeyLength = 0; } else { sortKeyLength = StringHelper.sortKeyLength(previous.get(), term); } offset += sortKeyLength; data.writeBytes(term.bytes, term.offset, sortKeyLength); } else if ((ord & Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == Lucene80DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) { previous.copyBytes(term); } ++ord; } writer.add(offset); writer.finish(); meta.writeLong(start); meta.writeLong(data.getFilePointer() - start); start = data.getFilePointer(); addressBuffer.copyTo(data); meta.writeLong(start); meta.writeLong(data.getFilePointer() - start); } }
Example 8
Source File: TestDocumentsWriterDeleteQueue.java From lucene-solr with Apache License 2.0 | 4 votes |
public void testStressDeleteQueue() throws Exception { DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(null); Set<Term> uniqueValues = new HashSet<>(); final int size = 10000 + random().nextInt(500) * RANDOM_MULTIPLIER; Integer[] ids = new Integer[size]; for (int i = 0; i < ids.length; i++) { ids[i] = random().nextInt(); uniqueValues.add(new Term("id", ids[i].toString())); } CountDownLatch latch = new CountDownLatch(1); AtomicInteger index = new AtomicInteger(0); final int numThreads = 2 + random().nextInt(5); UpdateThread[] threads = new UpdateThread[numThreads]; for (int i = 0; i < threads.length; i++) { threads[i] = new UpdateThread(queue, index, ids, latch); threads[i].start(); } latch.countDown(); for (int i = 0; i < threads.length; i++) { threads[i].join(); } for (UpdateThread updateThread : threads) { DeleteSlice slice = updateThread.slice; queue.updateSlice(slice); BufferedUpdates deletes = updateThread.deletes; slice.apply(deletes, BufferedUpdates.MAX_INT); assertEquals(uniqueValues, deletes.deleteTerms.keySet()); } queue.tryApplyGlobalSlice(); Set<Term> frozenSet = new HashSet<>(); BytesRefBuilder builder = new BytesRefBuilder(); TermIterator iter = queue.freezeGlobalBuffer(null).deleteTerms.iterator(); while (iter.next() != null) { builder.copyBytes(iter.bytes); frozenSet.add(new Term(iter.field(), builder.toBytesRef())); } assertEquals("num deletes must be 0 after freeze", 0, queue .numGlobalTermDeletes()); assertEquals(uniqueValues.size(), frozenSet.size()); assertEquals(uniqueValues, frozenSet); }