Java Code Examples for org.apache.lucene.store.DataInput#readByte()
The following examples show how to use
org.apache.lucene.store.DataInput#readByte() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Packed64.java From lucene-solr with Apache License 2.0 | 6 votes |
/** * Creates an array with content retrieved from the given DataInput. * @param in a DataInput, positioned at the start of Packed64-content. * @param valueCount the number of elements. * @param bitsPerValue the number of bits available for any given value. * @throws java.io.IOException if the values for the backing array could not * be retrieved. */ public Packed64(int packedIntsVersion, DataInput in, int valueCount, int bitsPerValue) throws IOException { super(valueCount, bitsPerValue); final PackedInts.Format format = PackedInts.Format.PACKED; final long byteCount = format.byteCount(packedIntsVersion, valueCount, bitsPerValue); // to know how much to read final int longCount = format.longCount(PackedInts.VERSION_CURRENT, valueCount, bitsPerValue); // to size the array blocks = new long[longCount]; // read as many longs as we can for (int i = 0; i < byteCount / 8; ++i) { blocks[i] = in.readLong(); } final int remaining = (int) (byteCount % 8); if (remaining != 0) { // read the last bytes long lastLong = 0; for (int i = 0; i < remaining; ++i) { lastLong |= (in.readByte() & 0xFFL) << (56 - i * 8); } blocks[blocks.length - 1] = lastLong; } maskRight = ~0L << (BLOCK_SIZE-bitsPerValue) >>> (BLOCK_SIZE-bitsPerValue); bpvMinusBlockSize = bitsPerValue - BLOCK_SIZE; }
Example 2
Source File: FSTTermOutputs.java From lucene-solr with Apache License 2.0 | 6 votes |
@Override public void skipOutput(DataInput in) throws IOException { int bits = in.readByte() & 0xff; int bit0 = bits & 1; int bit1 = bits & 2; int bytesSize = (bits >>> 2); if (bit0 > 0 && bytesSize == 0) { // determine extra length bytesSize = in.readVInt(); } if (bit0 > 0) { // bytes exists in.skipBytes(bytesSize); } if (bit1 > 0) { // stats exist int code = in.readVInt(); if (hasPos && (code & 1) == 0) { in.readVLong(); } } }
Example 3
Source File: SimpleTextUtil.java From lucene-solr with Apache License 2.0 | 6 votes |
public static void readLine(DataInput in, BytesRefBuilder scratch) throws IOException { int upto = 0; while(true) { byte b = in.readByte(); scratch.grow(1+upto); if (b == ESCAPE) { scratch.setByteAt(upto++, in.readByte()); } else { if (b == NEWLINE) { break; } else { scratch.setByteAt(upto++, b); } } } scratch.setLength(upto); }
Example 4
Source File: FreeTextSuggester.java From lucene-solr with Apache License 2.0 | 6 votes |
@Override public boolean load(DataInput input) throws IOException { CodecUtil.checkHeader(input, CODEC_NAME, VERSION_START, VERSION_START); count = input.readVLong(); byte separatorOrig = input.readByte(); if (separatorOrig != separator) { throw new IllegalStateException("separator=" + separator + " is incorrect: original model was built with separator=" + separatorOrig); } int gramsOrig = input.readVInt(); if (gramsOrig != grams) { throw new IllegalStateException("grams=" + grams + " is incorrect: original model was built with grams=" + gramsOrig); } totTokens = input.readVLong(); fst = new FST<>(input, input, PositiveIntOutputs.getSingleton()); return true; }
Example 5
Source File: CompressingStoredFieldsReader.java From lucene-solr with Apache License 2.0 | 6 votes |
/** * Reads a double in a variable-length format. Reads between one and * nine bytes. Small integral values typically take fewer bytes. */ static double readZDouble(DataInput in) throws IOException { int b = in.readByte() & 0xFF; if (b == 0xFF) { // negative value return Double.longBitsToDouble(in.readLong()); } else if (b == 0xFE) { // float return Float.intBitsToFloat(in.readInt()); } else if ((b & 0x80) != 0) { // small integer [-1..124] return (b & 0x7f) - 1; } else { // positive double long bits = ((long) b) << 56 | ((in.readInt() & 0xFFFFFFFFL) << 24) | ((in.readShort() & 0xFFFFL) << 8) | (in.readByte() & 0xFFL); return Double.longBitsToDouble(bits); } }
Example 6
Source File: TSTLookup.java From lucene-solr with Apache License 2.0 | 6 votes |
private void readRecursively(DataInput in, TernaryTreeNode node) throws IOException { node.splitchar = in.readString().charAt(0); byte mask = in.readByte(); if ((mask & HAS_TOKEN) != 0) { node.token = in.readString(); } if ((mask & HAS_VALUE) != 0) { node.val = Long.valueOf(in.readLong()); } if ((mask & LO_KID) != 0) { node.loKid = new TernaryTreeNode(); readRecursively(in, node.loKid); } if ((mask & EQ_KID) != 0) { node.eqKid = new TernaryTreeNode(); readRecursively(in, node.eqKid); } if ((mask & HI_KID) != 0) { node.hiKid = new TernaryTreeNode(); readRecursively(in, node.hiKid); } }
Example 7
Source File: BlockPackedReaderIterator.java From lucene-solr with Apache License 2.0 | 5 votes |
static long readVLong(DataInput in) throws IOException { byte b = in.readByte(); if (b >= 0) return b; long i = b & 0x7FL; b = in.readByte(); i |= (b & 0x7FL) << 7; if (b >= 0) return i; b = in.readByte(); i |= (b & 0x7FL) << 14; if (b >= 0) return i; b = in.readByte(); i |= (b & 0x7FL) << 21; if (b >= 0) return i; b = in.readByte(); i |= (b & 0x7FL) << 28; if (b >= 0) return i; b = in.readByte(); i |= (b & 0x7FL) << 35; if (b >= 0) return i; b = in.readByte(); i |= (b & 0x7FL) << 42; if (b >= 0) return i; b = in.readByte(); i |= (b & 0x7FL) << 49; if (b >= 0) return i; b = in.readByte(); i |= (b & 0xFFL) << 56; return i; }
Example 8
Source File: XAnalyzingSuggester.java From Elasticsearch with Apache License 2.0 | 5 votes |
@Override public boolean load(InputStream input) throws IOException { DataInput dataIn = new InputStreamDataInput(input); try { this.fst = new FST<>(dataIn, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton())); maxAnalyzedPathsForOneInput = dataIn.readVInt(); hasPayloads = dataIn.readByte() == 1; } finally { IOUtils.close(input); } return true; }
Example 9
Source File: FST.java From lucene-solr with Apache License 2.0 | 5 votes |
/** Reads one BYTE1/2/4 label from the provided {@link DataInput}. */ public int readLabel(DataInput in) throws IOException { final int v; if (inputType == INPUT_TYPE.BYTE1) { // Unsigned byte: v = in.readByte() & 0xFF; } else if (inputType == INPUT_TYPE.BYTE2) { // Unsigned short: v = in.readShort() & 0xFFFF; } else { v = in.readVInt(); } return v; }
Example 10
Source File: CodecUtil.java From lucene-solr with Apache License 2.0 | 5 votes |
/** Expert: just reads and verifies the suffix of an index header */ public static String checkIndexHeaderSuffix(DataInput in, String expectedSuffix) throws IOException { int suffixLength = in.readByte() & 0xFF; byte suffixBytes[] = new byte[suffixLength]; in.readBytes(suffixBytes, 0, suffixBytes.length); String suffix = new String(suffixBytes, 0, suffixBytes.length, StandardCharsets.UTF_8); if (!suffix.equals(expectedSuffix)) { throw new CorruptIndexException("file mismatch, expected suffix=" + expectedSuffix + ", got=" + suffix, in); } return suffix; }
Example 11
Source File: CompressingStoredFieldsReader.java From lucene-solr with Apache License 2.0 | 5 votes |
/** * Reads a long in a variable-length format. Reads between one andCorePropLo * nine bytes. Small values typically take fewer bytes. */ static long readTLong(DataInput in) throws IOException { int header = in.readByte() & 0xFF; long bits = header & 0x1F; if ((header & 0x20) != 0) { // continuation bit bits |= in.readVLong() << 5; } long l = BitUtil.zigZagDecode(bits); switch (header & DAY_ENCODING) { case SECOND_ENCODING: l *= SECOND; break; case HOUR_ENCODING: l *= HOUR; break; case DAY_ENCODING: l *= DAY; break; case 0: // uncompressed break; default: throw new AssertionError(); } return l; }
Example 12
Source File: CompressingStoredFieldsReader.java From lucene-solr with Apache License 2.0 | 5 votes |
/** * Reads a float in a variable-length format. Reads between one and * five bytes. Small integral values typically take fewer bytes. */ static float readZFloat(DataInput in) throws IOException { int b = in.readByte() & 0xFF; if (b == 0xFF) { // negative value return Float.intBitsToFloat(in.readInt()); } else if ((b & 0x80) != 0) { // small integer [-1..125] return (b & 0x7f) - 1; } else { // positive float int bits = b << 24 | ((in.readShort() & 0xFFFF) << 8) | (in.readByte() & 0xFF); return Float.intBitsToFloat(bits); } }
Example 13
Source File: Rot13CypherTestUtil.java From lucene-solr with Apache License 2.0 | 5 votes |
public static byte[] decode(DataInput bytesInput, long length) throws IOException { length -= ENCODING_OFFSET; bytesInput.skipBytes(ENCODING_OFFSET); byte[] decodedBytes = new byte[Math.toIntExact(length)]; for (int i = 0; i < length; i++) { decodedBytes[i] = (byte)(bytesInput.readByte() - ENCODING_ROTATION); } return decodedBytes; }
Example 14
Source File: Rot13CypherTestUtil.java From lucene-solr with Apache License 2.0 | 5 votes |
public static byte[] encode(DataInput bytesInput, int length) throws IOException { byte[] encodedBytes = new byte[length + ENCODING_OFFSET]; for (int i = 0; i < length; i++) { encodedBytes[i + ENCODING_OFFSET] = (byte)(bytesInput.readByte() + ENCODING_ROTATION); } return encodedBytes; }
Example 15
Source File: AnalyzingSuggester.java From lucene-solr with Apache License 2.0 | 5 votes |
@Override public boolean load(DataInput input) throws IOException { count = input.readVLong(); this.fst = new FST<>(input, input, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton())); maxAnalyzedPathsForOneInput = input.readVInt(); hasPayloads = input.readByte() == 1; return true; }
Example 16
Source File: FSTTermOutputs.java From lucene-solr with Apache License 2.0 | 5 votes |
@Override public TermData read(DataInput in) throws IOException { byte[] bytes = null; int docFreq = 0; long totalTermFreq = -1; int bits = in.readByte() & 0xff; int bit0 = bits & 1; int bit1 = bits & 2; int bytesSize = (bits >>> 2); if (bit0 > 0 && bytesSize == 0) { // determine extra length bytesSize = in.readVInt(); } if (bit0 > 0) { // bytes exists bytes = new byte[bytesSize]; in.readBytes(bytes, 0, bytesSize); } if (bit1 > 0) { // stats exist int code = in.readVInt(); if (hasPos) { totalTermFreq = docFreq = code >>> 1; if ((code & 1) == 0) { totalTermFreq += in.readVLong(); } } else { docFreq = code; } } return new TermData(bytes, docFreq, totalTermFreq); }
Example 17
Source File: SimpleTransLog.java From lucene-solr with Apache License 2.0 | 5 votes |
private String readNullableString(DataInput in) throws IOException { byte b = in.readByte(); if (b == 0) { return null; } else if (b == 1) { return in.readString(); } else { throw new CorruptIndexException("invalid string lead byte " + b, in); } }
Example 18
Source File: XAnalyzingSuggester.java From Elasticsearch with Apache License 2.0 | 5 votes |
@Override public boolean load(DataInput input) throws IOException { count = input.readVLong(); this.fst = new FST<>(input, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton())); maxAnalyzedPathsForOneInput = input.readVInt(); hasPayloads = input.readByte() == 1; return true; }
Example 19
Source File: SimplePrimaryNode.java From lucene-solr with Apache License 2.0 | 4 votes |
private void handleIndexing(Socket socket, AtomicBoolean stop, InputStream is, DataInput in, DataOutput out, BufferedOutputStream bos) throws IOException, InterruptedException { Thread.currentThread().setName("indexing"); message("start handling indexing socket=" + socket); while (true) { while (true) { if (is.available() > 0) { break; } if (stop.get()) { return; } Thread.sleep(10); } byte cmd; try { cmd = in.readByte(); } catch (EOFException eofe) { // done return; } //message("INDEXING OP " + cmd); if (cmd == CMD_ADD_DOC) { handleAddDocument(in, out); out.writeByte((byte) 1); bos.flush(); } else if (cmd == CMD_UPDATE_DOC) { handleUpdateDocument(in, out); out.writeByte((byte) 1); bos.flush(); } else if (cmd == CMD_DELETE_DOC) { handleDeleteDocument(in, out); out.writeByte((byte) 1); bos.flush(); } else if (cmd == CMD_DELETE_ALL_DOCS) { writer.deleteAll(); out.writeByte((byte) 1); bos.flush(); } else if (cmd == CMD_FORCE_MERGE) { writer.forceMerge(1); out.writeByte((byte) 1); bos.flush(); } else if (cmd == CMD_INDEXING_DONE) { out.writeByte((byte) 1); bos.flush(); break; } else { throw new IllegalArgumentException("cmd must be add, update or delete; got " + cmd); } } }
Example 20
Source File: SimplePrimaryNode.java From lucene-solr with Apache License 2.0 | 4 votes |
/** Called when another node (replica) wants to copy files from us */ private boolean handleFetchFiles(Random random, Socket socket, DataInput destIn, DataOutput destOut, BufferedOutputStream bos) throws IOException { Thread.currentThread().setName("send"); int replicaID = destIn.readVInt(); message("top: start fetch for R" + replicaID + " socket=" + socket); byte b = destIn.readByte(); CopyState copyState; if (b == 0) { // Caller already has CopyState copyState = null; } else if (b == 1) { // Caller does not have CopyState; we pull the latest one: copyState = getCopyState(); Thread.currentThread().setName("send-R" + replicaID + "-" + copyState.version); } else { // Protocol error: throw new IllegalArgumentException("invalid CopyState byte=" + b); } try { if (copyState != null) { // Serialize CopyState on the wire to the client: writeCopyState(copyState, destOut); bos.flush(); } byte[] buffer = new byte[16384]; int fileCount = 0; long totBytesSent = 0; while (true) { byte done = destIn.readByte(); if (done == 1) { break; } else if (done != 0) { throw new IllegalArgumentException("expected 0 or 1 byte but got " + done); } // Name of the file the replica wants us to send: String fileName = destIn.readString(); // Starting offset in the file we should start sending bytes from: long fpStart = destIn.readVLong(); try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) { long len = in.length(); //message("fetch " + fileName + ": send len=" + len); destOut.writeVLong(len); in.seek(fpStart); long upto = fpStart; while (upto < len) { int chunk = (int) Math.min(buffer.length, (len-upto)); in.readBytes(buffer, 0, chunk); if (doFlipBitsDuringCopy) { if (random.nextInt(3000) == 17 && bitFlipped.contains(fileName) == false) { bitFlipped.add(fileName); message("file " + fileName + " to R" + replicaID + ": now randomly flipping a bit at byte=" + upto); int x = random.nextInt(chunk); int bit = random.nextInt(8); buffer[x] ^= 1 << bit; } } destOut.writeBytes(buffer, 0, chunk); upto += chunk; totBytesSent += chunk; } } fileCount++; } message("top: done fetch files for R" + replicaID + ": sent " + fileCount + " files; sent " + totBytesSent + " bytes"); } catch (Throwable t) { message("top: exception during fetch: " + t.getMessage() + "; now close socket"); socket.close(); return false; } finally { if (copyState != null) { message("top: fetch: now release CopyState"); releaseCopyState(copyState); } } return true; }