Java Code Examples for org.apache.lucene.store.DataOutput#writeBytes()
The following examples show how to use
org.apache.lucene.store.DataOutput#writeBytes() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CharacterDefinitionWriter.java From lucene-solr with Apache License 2.0 | 6 votes |
public void write(Path baseDir) throws IOException { Path path = baseDir.resolve(CharacterDefinition.class.getName().replace('.', '/') + CharacterDefinition.FILENAME_SUFFIX); Files.createDirectories(path.getParent()); try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))){ final DataOutput out = new OutputStreamDataOutput(os); CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION); out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length); for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) { final byte b = (byte) ( (invokeMap[i] ? 0x01 : 0x00) | (groupMap[i] ? 0x02 : 0x00) ); out.writeByte(b); } } }
Example 2
Source File: CharacterDefinitionWriter.java From lucene-solr with Apache License 2.0 | 6 votes |
public void write(Path baseDir) throws IOException { Path path = baseDir.resolve(CharacterDefinition.class.getName().replace('.', '/') + CharacterDefinition.FILENAME_SUFFIX); Files.createDirectories(path.getParent()); try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))){ final DataOutput out = new OutputStreamDataOutput(os); CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION); out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length); for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) { final byte b = (byte) ( (invokeMap[i] ? 0x01 : 0x00) | (groupMap[i] ? 0x02 : 0x00) ); out.writeByte(b); } } }
Example 3
Source File: UniformSplitRot13PostingsFormat.java From lucene-solr with Apache License 2.0 | 6 votes |
protected BlockEncoder getBlockEncoder() { return (blockBytes, length) -> { byte[] encodedBytes = Rot13CypherTestUtil.encode(blockBytes, Math.toIntExact(length)); return new BlockEncoder.WritableBytes() { @Override public long size() { return encodedBytes.length; } @Override public void writeTo(DataOutput dataOutput) throws IOException { encoderCalled = true; dataOutput.writeBytes(encodedBytes, 0, encodedBytes.length); } }; }; }
Example 4
Source File: Rot13CypherTestUtil.java From lucene-solr with Apache License 2.0 | 6 votes |
public static BlockEncoder getBlockEncoder() { return (blockBytes, length) -> { byte[] encodedBytes = Rot13CypherTestUtil.encode(blockBytes, Math.toIntExact(length)); return new BlockEncoder.WritableBytes() { @Override public long size() { return encodedBytes.length; } @Override public void writeTo(DataOutput dataOutput) throws IOException { dataOutput.writeBytes(encodedBytes, 0, encodedBytes.length); } }; }; }
Example 5
Source File: SortingLeafReader.java From lucene-solr with Apache License 2.0 | 6 votes |
private void addPositions(final PostingsEnum in, final DataOutput out) throws IOException { int freq = in.freq(); out.writeVInt(freq); int previousPosition = 0; int previousEndOffset = 0; for (int i = 0; i < freq; i++) { final int pos = in.nextPosition(); final BytesRef payload = in.getPayload(); // The low-order bit of token is set only if there is a payload, the // previous bits are the delta-encoded position. final int token = (pos - previousPosition) << 1 | (payload == null ? 0 : 1); out.writeVInt(token); previousPosition = pos; if (storeOffsets) { // don't encode offsets if they are not stored final int startOffset = in.startOffset(); final int endOffset = in.endOffset(); out.writeVInt(startOffset - previousEndOffset); out.writeVInt(endOffset - startOffset); previousEndOffset = endOffset; } if (payload != null) { out.writeVInt(payload.length); out.writeBytes(payload.bytes, payload.offset, payload.length); } } }
Example 6
Source File: SimplePrimaryNode.java From lucene-solr with Apache License 2.0 | 5 votes |
/** Pushes CopyState on the wire */ private static void writeCopyState(CopyState state, DataOutput out) throws IOException { // TODO (opto): we could encode to byte[] once when we created the copyState, and then just send same byts to all replicas... out.writeVInt(state.infosBytes.length); out.writeBytes(state.infosBytes, 0, state.infosBytes.length); out.writeVLong(state.gen); out.writeVLong(state.version); SimpleServer.writeFilesMetaData(out, state.files); out.writeVInt(state.completedMergeFiles.size()); for(String fileName : state.completedMergeFiles) { out.writeString(fileName); } out.writeVLong(state.primaryGen); }
Example 7
Source File: SimpleServer.java From lucene-solr with Apache License 2.0 | 5 votes |
static void writeFilesMetaData(DataOutput out, Map<String,FileMetaData> files) throws IOException { out.writeVInt(files.size()); for(Map.Entry<String,FileMetaData> ent : files.entrySet()) { out.writeString(ent.getKey()); FileMetaData fmd = ent.getValue(); out.writeVLong(fmd.length); out.writeVLong(fmd.checksum); out.writeVInt(fmd.header.length); out.writeBytes(fmd.header, 0, fmd.header.length); out.writeVInt(fmd.footer.length); out.writeBytes(fmd.footer, 0, fmd.footer.length); } }
Example 8
Source File: FSTOrdsOutputs.java From lucene-solr with Apache License 2.0 | 5 votes |
@Override public void write(Output prefix, DataOutput out) throws IOException { out.writeVInt(prefix.bytes.length); out.writeBytes(prefix.bytes.bytes, prefix.bytes.offset, prefix.bytes.length); out.writeVLong(prefix.startOrd); out.writeVLong(prefix.endOrd); }
Example 9
Source File: BKDWriter.java From lucene-solr with Apache License 2.0 | 5 votes |
private void writeCommonPrefixes(DataOutput out, int[] commonPrefixes, byte[] packedValue) throws IOException { for(int dim=0;dim<numDataDims;dim++) { out.writeVInt(commonPrefixes[dim]); //System.out.println(commonPrefixes[dim] + " of " + bytesPerDim); out.writeBytes(packedValue, dim*bytesPerDim, commonPrefixes[dim]); } }
Example 10
Source File: BKDWriter.java From lucene-solr with Apache License 2.0 | 5 votes |
private void writeActualBounds(DataOutput out, int[] commonPrefixLengths, int count, IntFunction<BytesRef> packedValues) throws IOException { for (int dim = 0; dim < numIndexDims; ++dim) { int commonPrefixLength = commonPrefixLengths[dim]; int suffixLength = bytesPerDim - commonPrefixLength; if (suffixLength > 0) { BytesRef[] minMax = computeMinMax(count, packedValues, dim * bytesPerDim + commonPrefixLength, suffixLength); BytesRef min = minMax[0]; BytesRef max = minMax[1]; out.writeBytes(min.bytes, min.offset, min.length); out.writeBytes(max.bytes, max.offset, max.length); } } }
Example 11
Source File: CodecUtil.java From lucene-solr with Apache License 2.0 | 5 votes |
/** * Expert: verifies the incoming {@link IndexInput} has an index header * and that its segment ID matches the expected one, and then copies * that index header into the provided {@link DataOutput}. This is * useful when building compound files. * * @param in Input stream, positioned at the point where the * index header was previously written. Typically this is located * at the beginning of the file. * @param out Output stream, where the header will be copied to. * @param expectedID Expected segment ID * @throws CorruptIndexException If the first four bytes are not * {@link #CODEC_MAGIC}, or if the <code>expectedID</code> * does not match. * @throws IOException If there is an I/O error reading from the underlying medium. * * @lucene.internal */ public static void verifyAndCopyIndexHeader(IndexInput in, DataOutput out, byte[] expectedID) throws IOException { // make sure it's large enough to have a header and footer if (in.length() < footerLength() + headerLength("")) { throw new CorruptIndexException("compound sub-files must have a valid codec header and footer: file is too small (" + in.length() + " bytes)", in); } int actualHeader = in.readInt(); if (actualHeader != CODEC_MAGIC) { throw new CorruptIndexException("compound sub-files must have a valid codec header and footer: codec header mismatch: actual header=" + actualHeader + " vs expected header=" + CodecUtil.CODEC_MAGIC, in); } // we can't verify these, so we pass-through: String codec = in.readString(); int version = in.readInt(); // verify id: checkIndexHeaderID(in, expectedID); // we can't verify extension either, so we pass-through: int suffixLength = in.readByte() & 0xFF; byte[] suffixBytes = new byte[suffixLength]; in.readBytes(suffixBytes, 0, suffixLength); // now write the header we just verified out.writeInt(CodecUtil.CODEC_MAGIC); out.writeString(codec); out.writeInt(version); out.writeBytes(expectedID, 0, expectedID.length); out.writeByte((byte) suffixLength); out.writeBytes(suffixBytes, 0, suffixLength); }
Example 12
Source File: BKDWriter.java From lucene-solr with Apache License 2.0 | 5 votes |
private void writeLowCardinalityLeafBlockPackedValues(DataOutput out, int[] commonPrefixLengths, int count, IntFunction<BytesRef> packedValues) throws IOException { if (numIndexDims != 1) { writeActualBounds(out, commonPrefixLengths, count, packedValues); } BytesRef value = packedValues.apply(0); System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength); int cardinality = 1; for (int i = 1; i < count; i++) { value = packedValues.apply(i); for(int dim = 0; dim < numDataDims; dim++) { final int start = dim * bytesPerDim + commonPrefixLengths[dim]; final int end = dim * bytesPerDim + bytesPerDim; if (Arrays.mismatch(value.bytes, value.offset + start, value.offset + end, scratch1, start, end) != -1) { out.writeVInt(cardinality); for (int j = 0; j < numDataDims; j++) { out.writeBytes(scratch1, j * bytesPerDim + commonPrefixLengths[j], bytesPerDim - commonPrefixLengths[j]); } System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength); cardinality = 1; break; } else if (dim == numDataDims - 1){ cardinality++; } } } out.writeVInt(cardinality); for (int i = 0; i < numDataDims; i++) { out.writeBytes(scratch1, i * bytesPerDim + commonPrefixLengths[i], bytesPerDim - commonPrefixLengths[i]); } }
Example 13
Source File: LZ4.java From lucene-solr with Apache License 2.0 | 5 votes |
private static void encodeLiterals(byte[] bytes, int token, int anchor, int literalLen, DataOutput out) throws IOException { out.writeByte((byte) token); // encode literal length if (literalLen >= 0x0F) { encodeLen(literalLen - 0x0F, out); } // encode literals out.writeBytes(bytes, anchor, literalLen); }
Example 14
Source File: ByteArrayPrimitive.java From incubator-retired-blur with Apache License 2.0 | 4 votes |
@Override public void writeBytes(DataOutput out, int offset, int length) throws IOException { out.writeBytes(_bytes, offset, length); }
Example 15
Source File: BytesStore.java From lucene-solr with Apache License 2.0 | 4 votes |
/** Writes all of our bytes to the target {@link DataOutput}. */ public void writeTo(DataOutput out) throws IOException { for(byte[] block : blocks) { out.writeBytes(block, 0, block.length); } }
Example 16
Source File: DocValuesUpdate.java From lucene-solr with Apache License 2.0 | 4 votes |
@Override void writeTo(DataOutput out) throws IOException { assert hasValue; out.writeVInt(value.length); out.writeBytes(value.bytes, value.offset, value.length); }
Example 17
Source File: BlockTreeTermsWriter.java From lucene-solr with Apache License 2.0 | 4 votes |
private static void writeBytesRef(DataOutput out, BytesRef bytes) throws IOException { out.writeVInt(bytes.length); out.writeBytes(bytes.bytes, bytes.offset, bytes.length); }
Example 18
Source File: SimplePrimaryNode.java From lucene-solr with Apache License 2.0 | 4 votes |
/** Called when another node (replica) wants to copy files from us */ private boolean handleFetchFiles(Random random, Socket socket, DataInput destIn, DataOutput destOut, BufferedOutputStream bos) throws IOException { Thread.currentThread().setName("send"); int replicaID = destIn.readVInt(); message("top: start fetch for R" + replicaID + " socket=" + socket); byte b = destIn.readByte(); CopyState copyState; if (b == 0) { // Caller already has CopyState copyState = null; } else if (b == 1) { // Caller does not have CopyState; we pull the latest one: copyState = getCopyState(); Thread.currentThread().setName("send-R" + replicaID + "-" + copyState.version); } else { // Protocol error: throw new IllegalArgumentException("invalid CopyState byte=" + b); } try { if (copyState != null) { // Serialize CopyState on the wire to the client: writeCopyState(copyState, destOut); bos.flush(); } byte[] buffer = new byte[16384]; int fileCount = 0; long totBytesSent = 0; while (true) { byte done = destIn.readByte(); if (done == 1) { break; } else if (done != 0) { throw new IllegalArgumentException("expected 0 or 1 byte but got " + done); } // Name of the file the replica wants us to send: String fileName = destIn.readString(); // Starting offset in the file we should start sending bytes from: long fpStart = destIn.readVLong(); try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) { long len = in.length(); //message("fetch " + fileName + ": send len=" + len); destOut.writeVLong(len); in.seek(fpStart); long upto = fpStart; while (upto < len) { int chunk = (int) Math.min(buffer.length, (len-upto)); in.readBytes(buffer, 0, chunk); if (doFlipBitsDuringCopy) { if (random.nextInt(3000) == 17 && bitFlipped.contains(fileName) == false) { bitFlipped.add(fileName); message("file " + fileName + " to R" + replicaID + ": now randomly flipping a bit at byte=" + upto); int x = random.nextInt(chunk); int bit = random.nextInt(8); buffer[x] ^= 1 << bit; } } destOut.writeBytes(buffer, 0, chunk); upto += chunk; totBytesSent += chunk; } } fileCount++; } message("top: done fetch files for R" + replicaID + ": sent " + fileCount + " files; sent " + totBytesSent + " bytes"); } catch (Throwable t) { message("top: exception during fetch: " + t.getMessage() + "; now close socket"); socket.close(); return false; } finally { if (copyState != null) { message("top: fetch: now release CopyState"); releaseCopyState(copyState); } } return true; }
Example 19
Source File: CodecUtil.java From lucene-solr with Apache License 2.0 | 3 votes |
/** * Writes a codec header for an index file, which records both a string to * identify the format of the file, a version number, and data to identify * the file instance (ID and auxiliary suffix such as generation). * <p> * This header can be parsed and validated with * {@link #checkIndexHeader(DataInput, String, int, int, byte[], String) checkIndexHeader()}. * <p> * IndexHeader --> CodecHeader,ObjectID,ObjectSuffix * <ul> * <li>CodecHeader --> {@link #writeHeader} * <li>ObjectID --> {@link DataOutput#writeByte byte}<sup>16</sup> * <li>ObjectSuffix --> SuffixLength,SuffixBytes * <li>SuffixLength --> {@link DataOutput#writeByte byte} * <li>SuffixBytes --> {@link DataOutput#writeByte byte}<sup>SuffixLength</sup> * </ul> * <p> * Note that the length of an index header depends only upon the * name of the codec and suffix, so this length can be computed at any time * with {@link #indexHeaderLength(String,String)}. * * @param out Output stream * @param codec String to identify the format of this file. It should be simple ASCII, * less than 128 characters in length. * @param id Unique identifier for this particular file instance. * @param suffix auxiliary suffix information for the file. It should be simple ASCII, * less than 256 characters in length. * @param version Version number * @throws IOException If there is an I/O error writing to the underlying medium. * @throws IllegalArgumentException If the codec name is not simple ASCII, or * is more than 127 characters in length, or if id is invalid, * or if the suffix is not simple ASCII, or more than 255 characters * in length. */ public static void writeIndexHeader(DataOutput out, String codec, int version, byte[] id, String suffix) throws IOException { if (id.length != StringHelper.ID_LENGTH) { throw new IllegalArgumentException("Invalid id: " + StringHelper.idToString(id)); } writeHeader(out, codec, version); out.writeBytes(id, 0, id.length); BytesRef suffixBytes = new BytesRef(suffix); if (suffixBytes.length != suffix.length() || suffixBytes.length >= 256) { throw new IllegalArgumentException("suffix must be simple ASCII, less than 256 characters in length [got " + suffix + "]"); } out.writeByte((byte) suffixBytes.length); out.writeBytes(suffixBytes.bytes, suffixBytes.offset, suffixBytes.length); }