Java Code Examples for org.apache.lucene.codecs.CodecUtil#writeHeader()

The following examples show how to use org.apache.lucene.codecs.CodecUtil#writeHeader() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ConnectionCostsWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void write(Path baseDir) throws IOException {
  Files.createDirectories(baseDir);
  String fileName = ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX;
  try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName));
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);
    out.writeVInt(forwardSize);
    out.writeVInt(backwardSize);
    int last = 0;
    for (int i = 0; i < costs.limit() / 2; i++) {
      short cost = costs.getShort(i * 2);
      int delta = (int) cost - last;
      out.writeZInt(delta);
      last = cost;
    }
  }
}
 
Example 2
Source File: Store.java    From crate with Apache License 2.0 6 votes vote down vote up
/**
 * Marks this store as corrupted. This method writes a {@code corrupted_${uuid}} file containing the given exception
 * message. If a store contains a {@code corrupted_${uuid}} file {@link #isMarkedCorrupted()} will return <code>true</code>.
 */
public void markStoreCorrupted(IOException exception) throws IOException {
    ensureOpen();
    if (!isMarkedCorrupted()) {
        String uuid = CORRUPTED + UUIDs.randomBase64UUID();
        try (IndexOutput output = this.directory().createOutput(uuid, IOContext.DEFAULT)) {
            CodecUtil.writeHeader(output, CODEC, VERSION);
            BytesStreamOutput out = new BytesStreamOutput();
            out.writeException(exception);
            BytesReference bytes = out.bytes();
            output.writeVInt(bytes.length());
            BytesRef ref = bytes.toBytesRef();
            output.writeBytes(ref.bytes, ref.offset, ref.length);
            CodecUtil.writeFooter(output);
        } catch (IOException ex) {
            logger.warn("Can't mark store as corrupted", ex);
        }
        directory().sync(Collections.singleton(uuid));
    }
}
 
Example 3
Source File: DiskDocValuesConsumer.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
public DiskDocValuesConsumer(SegmentWriteState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
  boolean success = false;
  try {
    String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
    data = state.directory.createOutput(dataName, state.context);
    CodecUtil.writeHeader(data, dataCodec, DiskDocValuesFormat.VERSION_CURRENT);
    String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
    meta = state.directory.createOutput(metaName, state.context);
    CodecUtil.writeHeader(meta, metaCodec, DiskDocValuesFormat.VERSION_CURRENT);
    maxDoc = state.segmentInfo.getDocCount();
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(this);
    }
  }
}
 
Example 4
Source File: FieldsIndexWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
FieldsIndexWriter(Directory dir, String name, String suffix, String extension,
    String codecName, byte[] id, int blockShift, IOContext ioContext) throws IOException {
  this.dir = dir;
  this.name = name;
  this.suffix = suffix;
  this.extension = extension;
  this.codecName = codecName;
  this.id = id;
  this.blockShift = blockShift;
  this.ioContext = ioContext;
  this.docsOut = dir.createTempOutput(name, codecName + "-doc_ids", ioContext);
  boolean success = false;
  try {
    CodecUtil.writeHeader(docsOut, codecName + "Docs", VERSION_CURRENT);
    filePointersOut = dir.createTempOutput(name, codecName + "file_pointers", ioContext);
    CodecUtil.writeHeader(filePointersOut, codecName + "FilePointers", VERSION_CURRENT);
    success = true;
  } finally {
    if (success == false) {
      close();
    }
  }
}
 
Example 5
Source File: ChecksumBlobStoreFormat.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * Writes blob in atomic manner without resolving the blobName using using {@link #blobName} method.
 * <p>
 * The blob will be compressed and checksum will be written if required.
 *
 * @param obj           object to be serialized
 * @param blobContainer blob container
 * @param blobName          blob name
 */
protected void writeBlob(T obj, BlobContainer blobContainer, String blobName) throws IOException {
    BytesReference bytes = write(obj);
    try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
        final String resourceDesc = "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")";
        try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, byteArrayOutputStream, BUFFER_SIZE)) {
            CodecUtil.writeHeader(indexOutput, codec, VERSION);
            try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) {
                @Override
                public void close() throws IOException {
                    // this is important since some of the XContentBuilders write bytes on close.
                    // in order to write the footer we need to prevent closing the actual index input.
                } }) {
                bytes.writeTo(indexOutputOutputStream);
            }
            CodecUtil.writeFooter(indexOutput);
        }
        blobContainer.writeBlob(blobName, new BytesArray(byteArrayOutputStream.toByteArray()));
    }
}
 
Example 6
Source File: CharacterDefinitionWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void write(Path baseDir) throws IOException {
  Path path = baseDir.resolve(CharacterDefinition.class.getName().replace('.', '/') + CharacterDefinition.FILENAME_SUFFIX);
  Files.createDirectories(path.getParent());
  try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))){
    final DataOutput out = new OutputStreamDataOutput(os);
    CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);
    out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
    for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {
      final byte b = (byte) (
        (invokeMap[i] ? 0x01 : 0x00) | 
        (groupMap[i] ? 0x02 : 0x00)
      );
      out.writeByte(b);
    }
  }
}
 
Example 7
Source File: BinaryDictionaryWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void writePosDict(Path path) throws IOException {
  Files.createDirectories(path.getParent());
  try (OutputStream os = Files.newOutputStream(path);
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);
    out.writeVInt(posDict.size());
    for (String s : posDict) {
      if (s == null) {
        out.writeByte((byte) POS.Tag.UNKNOWN.ordinal());
      } else {
        String[] data = CSVUtil.parse(s);
        if (data.length != 2) {
          throw new IllegalArgumentException("Malformed pos/inflection: " + s + "; expected 2 characters");
        }
        out.writeByte((byte) POS.Tag.valueOf(data[0]).ordinal());
      }
    }
  }
}
 
Example 8
Source File: ConnectionCostsWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void write(Path baseDir) throws IOException {
  Files.createDirectories(baseDir);
  String fileName = ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX;
  try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName));
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);
    out.writeVInt(forwardSize);
    out.writeVInt(backwardSize);
    int last = 0;
    for (int i = 0; i < costs.limit() / 2; i++) {
      short cost = costs.getShort(i * 2);
      int delta = (int) cost - last;
      out.writeZInt(delta);
      last = cost;
    }
  }
}
 
Example 9
Source File: CharacterDefinitionWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void write(Path baseDir) throws IOException {
  Path path = baseDir.resolve(CharacterDefinition.class.getName().replace('.', '/') + CharacterDefinition.FILENAME_SUFFIX);
  Files.createDirectories(path.getParent());
  try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))){
    final DataOutput out = new OutputStreamDataOutput(os);
    CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);
    out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
    for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {
      final byte b = (byte) (
        (invokeMap[i] ? 0x01 : 0x00) | 
        (groupMap[i] ? 0x02 : 0x00)
      );
      out.writeByte(b);
    }
  }
}
 
Example 10
Source File: BinaryDictionaryWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void writePosDict(Path path) throws IOException {
  Files.createDirectories(path.getParent());
  try (OutputStream os = Files.newOutputStream(path);
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);
    out.writeVInt(posDict.size());
    for (String s : posDict) {
      if (s == null) {
        out.writeByte((byte)0);
        out.writeByte((byte)0);
        out.writeByte((byte)0);
      } else {
        String[] data = CSVUtil.parse(s);
        if (data.length != 3) {
          throw new IllegalArgumentException("Malformed pos/inflection: " + s + "; expected 3 characters");
        }
        out.writeString(data[0]);
        out.writeString(data[1]);
        out.writeString(data[2]);
      }
    }
  }
}
 
Example 11
Source File: TranslogWriter.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
public static TranslogWriter create(Type type, ShardId shardId, String translogUUID, long fileGeneration, Path file, Callback<ChannelReference> onClose, int bufferSize, ChannelFactory channelFactory) throws IOException {
    final BytesRef ref = new BytesRef(translogUUID);
    final int headerLength = getHeaderLength(ref.length);
    final FileChannel channel = channelFactory.open(file);
    try {
        // This OutputStreamDataOutput is intentionally not closed because
        // closing it will close the FileChannel
        final OutputStreamDataOutput out = new OutputStreamDataOutput(java.nio.channels.Channels.newOutputStream(channel));
        CodecUtil.writeHeader(out, TRANSLOG_CODEC, VERSION);
        out.writeInt(ref.length);
        out.writeBytes(ref.bytes, ref.offset, ref.length);
        channel.force(true);
        writeCheckpoint(headerLength, 0, file.getParent(), fileGeneration, StandardOpenOption.WRITE);
        final TranslogWriter writer = type.create(shardId, fileGeneration, new ChannelReference(file, fileGeneration, channel, onClose), bufferSize);
        return writer;
    } catch (Throwable throwable){
        // if we fail to bake the file-generation into the checkpoint we stick with the file and once we recover and that
        // file exists we remove it. We only apply this logic to the checkpoint.generation+1 any other file with a higher generation is an error condition
        IOUtils.closeWhileHandlingException(channel);
        throw throwable;
    }
}
 
Example 12
Source File: Completion090PostingsFormat.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
public CompletionFieldsConsumer(SegmentWriteState state) throws IOException {
    this.delegatesFieldsConsumer = delegatePostingsFormat.fieldsConsumer(state);
    String suggestFSTFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
    IndexOutput output = null;
    boolean success = false;
    try {
        output = state.directory.createOutput(suggestFSTFile, state.context);
        CodecUtil.writeHeader(output, CODEC_NAME, SUGGEST_VERSION_CURRENT);
        /*
         * we write the delegate postings format name so we can load it
         * without getting an instance in the ctor
         */
        output.writeString(delegatePostingsFormat.getName());
        output.writeString(writeProvider.getName());
        this.suggestFieldsConsumer = writeProvider.consumer(output);
        success = true;
    } finally {
        if (!success) {
            IOUtils.closeWhileHandlingException(output);
        }
    }
}
 
Example 13
Source File: FreeTextSuggester.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public boolean store(DataOutput output) throws IOException {
  CodecUtil.writeHeader(output, CODEC_NAME, VERSION_CURRENT);
  output.writeVLong(count);
  output.writeByte(separator);
  output.writeVInt(grams);
  output.writeVLong(totTokens);
  fst.save(output, output);
  return true;
}
 
Example 14
Source File: RAMOnlyPostingsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState writeState) throws IOException {
  final int id = nextID.getAndIncrement();

  // TODO -- ok to do this up front instead of
  // on close....?  should be ok?
  // Write our ID:
  final String idFileName = IndexFileNames.segmentFileName(writeState.segmentInfo.name, writeState.segmentSuffix, ID_EXTENSION);
  IndexOutput out = writeState.directory.createOutput(idFileName, writeState.context);
  boolean success = false;
  try {
    CodecUtil.writeHeader(out, RAM_ONLY_NAME, VERSION_LATEST);
    out.writeVInt(id);
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(out);
    } else {
      IOUtils.close(out);
    }
  }
  
  final RAMPostings postings = new RAMPostings();
  final RAMFieldsConsumer consumer = new RAMFieldsConsumer(writeState, postings);

  synchronized(state) {
    state.put(id, postings);
  }
  return consumer;
}
 
Example 15
Source File: Lucene80DocValuesConsumer.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public CompressedBinaryBlockWriter() throws IOException {
  tempBinaryOffsets = state.directory.createTempOutput(state.segmentInfo.name, "binary_pointers", state.context);
  boolean success = false;
  try {
    CodecUtil.writeHeader(tempBinaryOffsets, Lucene80DocValuesFormat.META_CODEC + "FilePointers", Lucene80DocValuesFormat.VERSION_CURRENT);
    blockAddressesStart = data.getFilePointer();
    success = true;
  } finally {
    if (success == false) {
      IOUtils.closeWhileHandlingException(this); //self-close because constructor caller can't 
    }
  }
}
 
Example 16
Source File: MetaDataStateFormat.java    From crate with Apache License 2.0 5 votes vote down vote up
private void writeStateToFirstLocation(final T state, Path stateLocation, Directory stateDir, String tmpFileName)
        throws WriteStateException {
    try {
        deleteFileIfExists(stateLocation, stateDir, tmpFileName);
        try (IndexOutput out = stateDir.createOutput(tmpFileName, IOContext.DEFAULT)) {
            CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION);
            out.writeInt(FORMAT.index());
            try (XContentBuilder builder = newXContentBuilder(FORMAT, new IndexOutputOutputStream(out) {
                @Override
                public void close() {
                    // this is important since some of the XContentBuilders write bytes on close.
                    // in order to write the footer we need to prevent closing the actual index input.
                }
            })) {
                builder.startObject();
                toXContent(builder, state);
                builder.endObject();
            }
            CodecUtil.writeFooter(out);
        }

        stateDir.sync(Collections.singleton(tmpFileName));
    } catch (Exception e) {
        throw new WriteStateException(false, "failed to write state to the first location tmp file " +
                stateLocation.resolve(tmpFileName), e);
    }
}
 
Example 17
Source File: PersistentSnapshotDeletionPolicy.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
synchronized private void persist() throws IOException {
  String fileName = SNAPSHOTS_PREFIX + nextWriteGen;
  IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT);
  boolean success = false;
  try {
    CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);   
    out.writeVInt(refCounts.size());
    for(Entry<Long,Integer> ent : refCounts.entrySet()) {
      out.writeVLong(ent.getKey());
      out.writeVInt(ent.getValue());
    }
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(out);
      IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
    } else {
      IOUtils.close(out);
    }
  }

  dir.sync(Collections.singletonList(fileName));
  
  if (nextWriteGen > 0) {
    String lastSaveFile = SNAPSHOTS_PREFIX + (nextWriteGen-1);
    // exception OK: likely it didn't exist
    IOUtils.deleteFilesIgnoringExceptions(dir, lastSaveFile);
  }

  nextWriteGen++;
}
 
Example 18
Source File: SolrSnapshotMetaDataManager.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private synchronized void persist() throws IOException {
  String fileName = SNAPSHOTS_PREFIX + nextWriteGen;
  IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT);
  boolean success = false;
  try {
    CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
    out.writeVInt(nameToDetailsMapping.size());
    for(Entry<String,SnapshotMetaData> ent : nameToDetailsMapping.entrySet()) {
      out.writeString(ent.getKey());
      out.writeString(ent.getValue().getIndexDirPath());
      out.writeVLong(ent.getValue().getGenerationNumber());
    }
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(out);
      IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
    } else {
      IOUtils.close(out);
    }
  }

  dir.sync(Collections.singletonList(fileName));

  if (nextWriteGen > 0) {
    String lastSaveFile = SNAPSHOTS_PREFIX + (nextWriteGen-1);
    // exception OK: likely it didn't exist
    IOUtils.deleteFilesIgnoringExceptions(dir, lastSaveFile);
  }

  nextWriteGen++;
}
 
Example 19
Source File: Blur022SegmentInfoWriter.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
@Override
public void write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(si.name, "", Blur022SegmentInfoFormat.SI_EXTENSION);
  si.addFile(fileName);

  final IndexOutput output = dir.createOutput(fileName, ioContext);

  boolean success = false;
  try {
    CodecUtil.writeHeader(output, Blur022SegmentInfoFormat.CODEC_NAME, Blur022SegmentInfoFormat.VERSION_CURRENT);
    output.writeString(si.getVersion());
    output.writeInt(si.getDocCount());

    output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
    output.writeStringStringMap(si.getDiagnostics());
    Map<String, String> attributes = si.attributes();
    TreeMap<String, String> newAttributes = new TreeMap<String, String>();
    if (attributes != null) {
      newAttributes.putAll(attributes);
    }
    newAttributes.put(Blur022StoredFieldsFormat.STORED_FIELDS_FORMAT_CHUNK_SIZE,
        Integer.toString(_compressionChunkSize));
    newAttributes.put(Blur022StoredFieldsFormat.STORED_FIELDS_FORMAT_COMPRESSION_MODE, _compressionMode);
    output.writeStringStringMap(newAttributes);
    output.writeStringSet(si.files());

    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(output);
      si.dir.deleteFile(fileName);
    } else {
      output.close();
    }
  }
}
 
Example 20
Source File: ChecksumBlobStoreFormat.java    From crate with Apache License 2.0 5 votes vote down vote up
private void writeTo(final T obj, final String blobName, final CheckedConsumer<BytesArray, IOException> consumer) throws IOException {
    final BytesReference bytes;
    try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) {
        if (compress) {
            try (StreamOutput compressedStreamOutput = CompressorFactory.COMPRESSOR.streamOutput(bytesStreamOutput)) {
                write(obj, compressedStreamOutput);
            }
        } else {
            write(obj, bytesStreamOutput);
        }
        bytes = bytesStreamOutput.bytes();
    }
    try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
        final String resourceDesc = "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")";
        try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, blobName, outputStream, BUFFER_SIZE)) {
            CodecUtil.writeHeader(indexOutput, codec, VERSION);
            try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) {
                @Override
                public void close() {
                    // this is important since some of the XContentBuilders write bytes on close.
                    // in order to write the footer we need to prevent closing the actual index input.
                }
            }) {
                bytes.writeTo(indexOutputOutputStream);
            }
            CodecUtil.writeFooter(indexOutput);
        }
        consumer.accept(new BytesArray(outputStream.toByteArray()));
    }
}