Java Code Examples for org.apache.lucene.codecs.CodecUtil#writeHeader()
The following examples show how to use
org.apache.lucene.codecs.CodecUtil#writeHeader() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ConnectionCostsWriter.java From lucene-solr with Apache License 2.0 | 6 votes |
public void write(Path baseDir) throws IOException { Files.createDirectories(baseDir); String fileName = ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX; try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName)); OutputStream bos = new BufferedOutputStream(os)) { final DataOutput out = new OutputStreamDataOutput(bos); CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION); out.writeVInt(forwardSize); out.writeVInt(backwardSize); int last = 0; for (int i = 0; i < costs.limit() / 2; i++) { short cost = costs.getShort(i * 2); int delta = (int) cost - last; out.writeZInt(delta); last = cost; } } }
Example 2
Source File: Store.java From crate with Apache License 2.0 | 6 votes |
/** * Marks this store as corrupted. This method writes a {@code corrupted_${uuid}} file containing the given exception * message. If a store contains a {@code corrupted_${uuid}} file {@link #isMarkedCorrupted()} will return <code>true</code>. */ public void markStoreCorrupted(IOException exception) throws IOException { ensureOpen(); if (!isMarkedCorrupted()) { String uuid = CORRUPTED + UUIDs.randomBase64UUID(); try (IndexOutput output = this.directory().createOutput(uuid, IOContext.DEFAULT)) { CodecUtil.writeHeader(output, CODEC, VERSION); BytesStreamOutput out = new BytesStreamOutput(); out.writeException(exception); BytesReference bytes = out.bytes(); output.writeVInt(bytes.length()); BytesRef ref = bytes.toBytesRef(); output.writeBytes(ref.bytes, ref.offset, ref.length); CodecUtil.writeFooter(output); } catch (IOException ex) { logger.warn("Can't mark store as corrupted", ex); } directory().sync(Collections.singleton(uuid)); } }
Example 3
Source File: DiskDocValuesConsumer.java From incubator-retired-blur with Apache License 2.0 | 6 votes |
public DiskDocValuesConsumer(SegmentWriteState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException { boolean success = false; try { String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension); data = state.directory.createOutput(dataName, state.context); CodecUtil.writeHeader(data, dataCodec, DiskDocValuesFormat.VERSION_CURRENT); String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension); meta = state.directory.createOutput(metaName, state.context); CodecUtil.writeHeader(meta, metaCodec, DiskDocValuesFormat.VERSION_CURRENT); maxDoc = state.segmentInfo.getDocCount(); success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(this); } } }
Example 4
Source File: FieldsIndexWriter.java From lucene-solr with Apache License 2.0 | 6 votes |
FieldsIndexWriter(Directory dir, String name, String suffix, String extension, String codecName, byte[] id, int blockShift, IOContext ioContext) throws IOException { this.dir = dir; this.name = name; this.suffix = suffix; this.extension = extension; this.codecName = codecName; this.id = id; this.blockShift = blockShift; this.ioContext = ioContext; this.docsOut = dir.createTempOutput(name, codecName + "-doc_ids", ioContext); boolean success = false; try { CodecUtil.writeHeader(docsOut, codecName + "Docs", VERSION_CURRENT); filePointersOut = dir.createTempOutput(name, codecName + "file_pointers", ioContext); CodecUtil.writeHeader(filePointersOut, codecName + "FilePointers", VERSION_CURRENT); success = true; } finally { if (success == false) { close(); } } }
Example 5
Source File: ChecksumBlobStoreFormat.java From Elasticsearch with Apache License 2.0 | 6 votes |
/** * Writes blob in atomic manner without resolving the blobName using using {@link #blobName} method. * <p> * The blob will be compressed and checksum will be written if required. * * @param obj object to be serialized * @param blobContainer blob container * @param blobName blob name */ protected void writeBlob(T obj, BlobContainer blobContainer, String blobName) throws IOException { BytesReference bytes = write(obj); try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { final String resourceDesc = "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")"; try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, byteArrayOutputStream, BUFFER_SIZE)) { CodecUtil.writeHeader(indexOutput, codec, VERSION); try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) { @Override public void close() throws IOException { // this is important since some of the XContentBuilders write bytes on close. // in order to write the footer we need to prevent closing the actual index input. } }) { bytes.writeTo(indexOutputOutputStream); } CodecUtil.writeFooter(indexOutput); } blobContainer.writeBlob(blobName, new BytesArray(byteArrayOutputStream.toByteArray())); } }
Example 6
Source File: CharacterDefinitionWriter.java From lucene-solr with Apache License 2.0 | 6 votes |
public void write(Path baseDir) throws IOException { Path path = baseDir.resolve(CharacterDefinition.class.getName().replace('.', '/') + CharacterDefinition.FILENAME_SUFFIX); Files.createDirectories(path.getParent()); try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))){ final DataOutput out = new OutputStreamDataOutput(os); CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION); out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length); for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) { final byte b = (byte) ( (invokeMap[i] ? 0x01 : 0x00) | (groupMap[i] ? 0x02 : 0x00) ); out.writeByte(b); } } }
Example 7
Source File: BinaryDictionaryWriter.java From lucene-solr with Apache License 2.0 | 6 votes |
private void writePosDict(Path path) throws IOException { Files.createDirectories(path.getParent()); try (OutputStream os = Files.newOutputStream(path); OutputStream bos = new BufferedOutputStream(os)) { final DataOutput out = new OutputStreamDataOutput(bos); CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION); out.writeVInt(posDict.size()); for (String s : posDict) { if (s == null) { out.writeByte((byte) POS.Tag.UNKNOWN.ordinal()); } else { String[] data = CSVUtil.parse(s); if (data.length != 2) { throw new IllegalArgumentException("Malformed pos/inflection: " + s + "; expected 2 characters"); } out.writeByte((byte) POS.Tag.valueOf(data[0]).ordinal()); } } } }
Example 8
Source File: ConnectionCostsWriter.java From lucene-solr with Apache License 2.0 | 6 votes |
public void write(Path baseDir) throws IOException { Files.createDirectories(baseDir); String fileName = ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX; try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName)); OutputStream bos = new BufferedOutputStream(os)) { final DataOutput out = new OutputStreamDataOutput(bos); CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION); out.writeVInt(forwardSize); out.writeVInt(backwardSize); int last = 0; for (int i = 0; i < costs.limit() / 2; i++) { short cost = costs.getShort(i * 2); int delta = (int) cost - last; out.writeZInt(delta); last = cost; } } }
Example 9
Source File: CharacterDefinitionWriter.java From lucene-solr with Apache License 2.0 | 6 votes |
public void write(Path baseDir) throws IOException { Path path = baseDir.resolve(CharacterDefinition.class.getName().replace('.', '/') + CharacterDefinition.FILENAME_SUFFIX); Files.createDirectories(path.getParent()); try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))){ final DataOutput out = new OutputStreamDataOutput(os); CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION); out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length); for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) { final byte b = (byte) ( (invokeMap[i] ? 0x01 : 0x00) | (groupMap[i] ? 0x02 : 0x00) ); out.writeByte(b); } } }
Example 10
Source File: BinaryDictionaryWriter.java From lucene-solr with Apache License 2.0 | 6 votes |
private void writePosDict(Path path) throws IOException { Files.createDirectories(path.getParent()); try (OutputStream os = Files.newOutputStream(path); OutputStream bos = new BufferedOutputStream(os)) { final DataOutput out = new OutputStreamDataOutput(bos); CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION); out.writeVInt(posDict.size()); for (String s : posDict) { if (s == null) { out.writeByte((byte)0); out.writeByte((byte)0); out.writeByte((byte)0); } else { String[] data = CSVUtil.parse(s); if (data.length != 3) { throw new IllegalArgumentException("Malformed pos/inflection: " + s + "; expected 3 characters"); } out.writeString(data[0]); out.writeString(data[1]); out.writeString(data[2]); } } } }
Example 11
Source File: TranslogWriter.java From Elasticsearch with Apache License 2.0 | 6 votes |
public static TranslogWriter create(Type type, ShardId shardId, String translogUUID, long fileGeneration, Path file, Callback<ChannelReference> onClose, int bufferSize, ChannelFactory channelFactory) throws IOException { final BytesRef ref = new BytesRef(translogUUID); final int headerLength = getHeaderLength(ref.length); final FileChannel channel = channelFactory.open(file); try { // This OutputStreamDataOutput is intentionally not closed because // closing it will close the FileChannel final OutputStreamDataOutput out = new OutputStreamDataOutput(java.nio.channels.Channels.newOutputStream(channel)); CodecUtil.writeHeader(out, TRANSLOG_CODEC, VERSION); out.writeInt(ref.length); out.writeBytes(ref.bytes, ref.offset, ref.length); channel.force(true); writeCheckpoint(headerLength, 0, file.getParent(), fileGeneration, StandardOpenOption.WRITE); final TranslogWriter writer = type.create(shardId, fileGeneration, new ChannelReference(file, fileGeneration, channel, onClose), bufferSize); return writer; } catch (Throwable throwable){ // if we fail to bake the file-generation into the checkpoint we stick with the file and once we recover and that // file exists we remove it. We only apply this logic to the checkpoint.generation+1 any other file with a higher generation is an error condition IOUtils.closeWhileHandlingException(channel); throw throwable; } }
Example 12
Source File: Completion090PostingsFormat.java From Elasticsearch with Apache License 2.0 | 6 votes |
public CompletionFieldsConsumer(SegmentWriteState state) throws IOException { this.delegatesFieldsConsumer = delegatePostingsFormat.fieldsConsumer(state); String suggestFSTFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION); IndexOutput output = null; boolean success = false; try { output = state.directory.createOutput(suggestFSTFile, state.context); CodecUtil.writeHeader(output, CODEC_NAME, SUGGEST_VERSION_CURRENT); /* * we write the delegate postings format name so we can load it * without getting an instance in the ctor */ output.writeString(delegatePostingsFormat.getName()); output.writeString(writeProvider.getName()); this.suggestFieldsConsumer = writeProvider.consumer(output); success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(output); } } }
Example 13
Source File: FreeTextSuggester.java From lucene-solr with Apache License 2.0 | 5 votes |
@Override public boolean store(DataOutput output) throws IOException { CodecUtil.writeHeader(output, CODEC_NAME, VERSION_CURRENT); output.writeVLong(count); output.writeByte(separator); output.writeVInt(grams); output.writeVLong(totTokens); fst.save(output, output); return true; }
Example 14
Source File: RAMOnlyPostingsFormat.java From lucene-solr with Apache License 2.0 | 5 votes |
@Override public FieldsConsumer fieldsConsumer(SegmentWriteState writeState) throws IOException { final int id = nextID.getAndIncrement(); // TODO -- ok to do this up front instead of // on close....? should be ok? // Write our ID: final String idFileName = IndexFileNames.segmentFileName(writeState.segmentInfo.name, writeState.segmentSuffix, ID_EXTENSION); IndexOutput out = writeState.directory.createOutput(idFileName, writeState.context); boolean success = false; try { CodecUtil.writeHeader(out, RAM_ONLY_NAME, VERSION_LATEST); out.writeVInt(id); success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(out); } else { IOUtils.close(out); } } final RAMPostings postings = new RAMPostings(); final RAMFieldsConsumer consumer = new RAMFieldsConsumer(writeState, postings); synchronized(state) { state.put(id, postings); } return consumer; }
Example 15
Source File: Lucene80DocValuesConsumer.java From lucene-solr with Apache License 2.0 | 5 votes |
public CompressedBinaryBlockWriter() throws IOException { tempBinaryOffsets = state.directory.createTempOutput(state.segmentInfo.name, "binary_pointers", state.context); boolean success = false; try { CodecUtil.writeHeader(tempBinaryOffsets, Lucene80DocValuesFormat.META_CODEC + "FilePointers", Lucene80DocValuesFormat.VERSION_CURRENT); blockAddressesStart = data.getFilePointer(); success = true; } finally { if (success == false) { IOUtils.closeWhileHandlingException(this); //self-close because constructor caller can't } } }
Example 16
Source File: MetaDataStateFormat.java From crate with Apache License 2.0 | 5 votes |
private void writeStateToFirstLocation(final T state, Path stateLocation, Directory stateDir, String tmpFileName) throws WriteStateException { try { deleteFileIfExists(stateLocation, stateDir, tmpFileName); try (IndexOutput out = stateDir.createOutput(tmpFileName, IOContext.DEFAULT)) { CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION); out.writeInt(FORMAT.index()); try (XContentBuilder builder = newXContentBuilder(FORMAT, new IndexOutputOutputStream(out) { @Override public void close() { // this is important since some of the XContentBuilders write bytes on close. // in order to write the footer we need to prevent closing the actual index input. } })) { builder.startObject(); toXContent(builder, state); builder.endObject(); } CodecUtil.writeFooter(out); } stateDir.sync(Collections.singleton(tmpFileName)); } catch (Exception e) { throw new WriteStateException(false, "failed to write state to the first location tmp file " + stateLocation.resolve(tmpFileName), e); } }
Example 17
Source File: PersistentSnapshotDeletionPolicy.java From lucene-solr with Apache License 2.0 | 5 votes |
synchronized private void persist() throws IOException { String fileName = SNAPSHOTS_PREFIX + nextWriteGen; IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT); boolean success = false; try { CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); out.writeVInt(refCounts.size()); for(Entry<Long,Integer> ent : refCounts.entrySet()) { out.writeVLong(ent.getKey()); out.writeVInt(ent.getValue()); } success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(out); IOUtils.deleteFilesIgnoringExceptions(dir, fileName); } else { IOUtils.close(out); } } dir.sync(Collections.singletonList(fileName)); if (nextWriteGen > 0) { String lastSaveFile = SNAPSHOTS_PREFIX + (nextWriteGen-1); // exception OK: likely it didn't exist IOUtils.deleteFilesIgnoringExceptions(dir, lastSaveFile); } nextWriteGen++; }
Example 18
Source File: SolrSnapshotMetaDataManager.java From lucene-solr with Apache License 2.0 | 5 votes |
private synchronized void persist() throws IOException { String fileName = SNAPSHOTS_PREFIX + nextWriteGen; IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT); boolean success = false; try { CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); out.writeVInt(nameToDetailsMapping.size()); for(Entry<String,SnapshotMetaData> ent : nameToDetailsMapping.entrySet()) { out.writeString(ent.getKey()); out.writeString(ent.getValue().getIndexDirPath()); out.writeVLong(ent.getValue().getGenerationNumber()); } success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(out); IOUtils.deleteFilesIgnoringExceptions(dir, fileName); } else { IOUtils.close(out); } } dir.sync(Collections.singletonList(fileName)); if (nextWriteGen > 0) { String lastSaveFile = SNAPSHOTS_PREFIX + (nextWriteGen-1); // exception OK: likely it didn't exist IOUtils.deleteFilesIgnoringExceptions(dir, lastSaveFile); } nextWriteGen++; }
Example 19
Source File: Blur022SegmentInfoWriter.java From incubator-retired-blur with Apache License 2.0 | 5 votes |
@Override public void write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) throws IOException { final String fileName = IndexFileNames.segmentFileName(si.name, "", Blur022SegmentInfoFormat.SI_EXTENSION); si.addFile(fileName); final IndexOutput output = dir.createOutput(fileName, ioContext); boolean success = false; try { CodecUtil.writeHeader(output, Blur022SegmentInfoFormat.CODEC_NAME, Blur022SegmentInfoFormat.VERSION_CURRENT); output.writeString(si.getVersion()); output.writeInt(si.getDocCount()); output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO)); output.writeStringStringMap(si.getDiagnostics()); Map<String, String> attributes = si.attributes(); TreeMap<String, String> newAttributes = new TreeMap<String, String>(); if (attributes != null) { newAttributes.putAll(attributes); } newAttributes.put(Blur022StoredFieldsFormat.STORED_FIELDS_FORMAT_CHUNK_SIZE, Integer.toString(_compressionChunkSize)); newAttributes.put(Blur022StoredFieldsFormat.STORED_FIELDS_FORMAT_COMPRESSION_MODE, _compressionMode); output.writeStringStringMap(newAttributes); output.writeStringSet(si.files()); success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(output); si.dir.deleteFile(fileName); } else { output.close(); } } }
Example 20
Source File: ChecksumBlobStoreFormat.java From crate with Apache License 2.0 | 5 votes |
private void writeTo(final T obj, final String blobName, final CheckedConsumer<BytesArray, IOException> consumer) throws IOException { final BytesReference bytes; try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) { if (compress) { try (StreamOutput compressedStreamOutput = CompressorFactory.COMPRESSOR.streamOutput(bytesStreamOutput)) { write(obj, compressedStreamOutput); } } else { write(obj, bytesStreamOutput); } bytes = bytesStreamOutput.bytes(); } try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { final String resourceDesc = "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")"; try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, blobName, outputStream, BUFFER_SIZE)) { CodecUtil.writeHeader(indexOutput, codec, VERSION); try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) { @Override public void close() { // this is important since some of the XContentBuilders write bytes on close. // in order to write the footer we need to prevent closing the actual index input. } }) { bytes.writeTo(indexOutputOutputStream); } CodecUtil.writeFooter(indexOutput); } consumer.accept(new BytesArray(outputStream.toByteArray())); } }