org.elasticsearch.indices.recovery.RecoveryState Java Examples

private void addChildImplementations(final RecoveryState recoveryState) {
    childImplementations.put(USED, new SimpleObjectExpression<Long>() {
        @Override
        public Long value() {
            return recoveryState.getIndex().totalBytes();
        }
    });
    childImplementations.put(REUSED, new SimpleObjectExpression<Long>() {
        @Override
        public Long value() {
            return recoveryState.getIndex().reusedBytes();
        }
    });
    childImplementations.put(RECOVERED, new SimpleObjectExpression<Long>() {
        @Override
        public Long value() {
            return recoveryState.getIndex().recoveredBytes();
        }
    });
    childImplementations.put(PERCENT, new SimpleObjectExpression<Float>() {
        @Override
        public Float value() {
            return recoveryState.getIndex().recoveredBytesPercent();
        }
    });
}
 

@Override
public void restoreShard(Store store, SnapshotId snapshotId, Version version, IndexId indexId, ShardId snapshotShardId,
                         RecoveryState recoveryState) {
    ShardId shardId = store.shardId();
    try {
        final BlobContainer container = shardContainer(indexId, snapshotShardId);
        BlobStoreIndexShardSnapshot snapshot = loadShardSnapshot(container, snapshotId);
        SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles());
        new FileRestoreContext(metadata.name(), shardId, snapshotId, recoveryState, BUFFER_SIZE) {
            @Override
            protected InputStream fileInputStream(BlobStoreIndexShardSnapshot.FileInfo fileInfo) {
                final InputStream dataBlobCompositeStream = new SlicedInputStream(fileInfo.numberOfParts()) {
                    @Override
                    protected InputStream openSlice(long slice) throws IOException {
                        return container.readBlob(fileInfo.partName(slice));
                    }
                };
                return restoreRateLimiter == null ? dataBlobCompositeStream
                    : new RateLimitingInputStream(dataBlobCompositeStream, restoreRateLimiter, restoreRateLimitingTimeInNanos::inc);
            }
        }.restore(snapshotFiles, store);
    } catch (Exception e) {
        throw new IndexShardRestoreFailedException(shardId, "failed to restore snapshot [" + snapshotId + "]", e);
    }
}
 

@Override
public IndexShard createShard(ShardRouting shardRouting, RecoveryState recoveryState, PeerRecoveryTargetService recoveryTargetService,
                              PeerRecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
                              Consumer<IndexShard.ShardFailure> onShardFailure,
                              Consumer<ShardId> globalCheckpointSyncer) throws IOException {
    ensureChangesAllowed();
    IndexService indexService = indexService(shardRouting.index());
    IndexShard indexShard = indexService.createShard(shardRouting, globalCheckpointSyncer);
    indexShard.addShardFailureCallback(onShardFailure);
    indexShard.startRecovery(recoveryState, recoveryTargetService, recoveryListener, repositoriesService,
        (type, mapping) -> {
            assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS :
                "mapping update consumer only required by local shards recovery";
            client.admin().indices().preparePutMapping()
                .setConcreteIndex(shardRouting.index()) // concrete index - no name clash, it uses uuid
                .setType(type)
                .setSource(mapping.source().string(), XContentType.JSON)
                .get();
        }, this);
    return indexShard;
}
 

/**
 * Marks the shard as recovering based on a recovery state, fails with exception is recovering is not allowed to be set.
 */
public IndexShardState markAsRecovering(String reason, RecoveryState recoveryState) throws IndexShardStartedException,
    IndexShardRelocatedException, IndexShardRecoveringException, IndexShardClosedException {
    synchronized (mutex) {
        if (state == IndexShardState.CLOSED) {
            throw new IndexShardClosedException(shardId);
        }
        if (state == IndexShardState.STARTED) {
            throw new IndexShardStartedException(shardId);
        }
        if (state == IndexShardState.RECOVERING) {
            throw new IndexShardRecoveringException(shardId);
        }
        if (state == IndexShardState.POST_RECOVERY) {
            throw new IndexShardRecoveringException(shardId);
        }
        this.recoveryState = recoveryState;
        return changeState(IndexShardState.RECOVERING, reason);
    }
}
 

public IndexShard postRecovery(String reason) throws IndexShardStartedException, IndexShardRelocatedException, IndexShardClosedException {
    synchronized (mutex) {
        if (state == IndexShardState.CLOSED) {
            throw new IndexShardClosedException(shardId);
        }
        if (state == IndexShardState.STARTED) {
            throw new IndexShardStartedException(shardId);
        }
        // we need to refresh again to expose all operations that were index until now. Otherwise
        // we may not expose operations that were indexed with a refresh listener that was immediately
        // responded to in addRefreshListener.
        getEngine().refresh("post_recovery");
        recoveryState.setStage(RecoveryState.Stage.DONE);
        changeState(IndexShardState.POST_RECOVERY, reason);
    }
    return this;
}
 

@Override
protected RecoveryResponse newResponse(RecoveryRequest request, int totalShards, int successfulShards, int failedShards, List<RecoveryState> responses, List<ShardOperationFailedException> shardFailures, ClusterState clusterState) {
    Map<String, List<RecoveryState>> shardResponses = Maps.newHashMap();
    for (RecoveryState recoveryState : responses) {
        if (recoveryState == null) {
            continue;
        }
        String indexName = recoveryState.getShardId().getIndex();
        if (!shardResponses.containsKey(indexName)) {
            shardResponses.put(indexName, new ArrayList<RecoveryState>());
        }
        if (request.activeOnly()) {
            if (recoveryState.getStage() != RecoveryState.Stage.DONE) {
                shardResponses.get(indexName).add(recoveryState);
            }
        } else {
            shardResponses.get(indexName).add(recoveryState);
        }
    }
    return new RecoveryResponse(totalShards, successfulShards, failedShards, request.detailed(), shardResponses, shardFailures);
}
 

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
    if (hasRecoveries()) {
        for (String index : shardRecoveryStates.keySet()) {
            List<RecoveryState> recoveryStates = shardRecoveryStates.get(index);
            if (recoveryStates == null || recoveryStates.size() == 0) {
                continue;
            }
            builder.startObject(index);
            builder.startArray("shards");
            for (RecoveryState recoveryState : recoveryStates) {
                builder.startObject();
                recoveryState.toXContent(builder, params);
                builder.endObject();
            }
            builder.endArray();
            builder.endObject();
        }
    }
    return builder;
}
 

@Override
protected RecoveryResponse newResponse(RecoveryRequest request, int totalShards, int successfulShards, int failedShards, List<RecoveryState> responses, List<DefaultShardOperationFailedException> shardFailures, ClusterState clusterState) {
    Map<String, List<RecoveryState>> shardResponses = new HashMap<>();
    for (RecoveryState recoveryState : responses) {
        if (recoveryState == null) {
            continue;
        }
        String indexName = recoveryState.getShardId().getIndexName();
        if (!shardResponses.containsKey(indexName)) {
            shardResponses.put(indexName, new ArrayList<>());
        }
        if (request.activeOnly()) {
            if (recoveryState.getStage() != RecoveryState.Stage.DONE) {
                shardResponses.get(indexName).add(recoveryState);
            }
        } else {
            shardResponses.get(indexName).add(recoveryState);
        }
    }
    return new RecoveryResponse(totalShards, successfulShards, failedShards, shardResponses, shardFailures);
}
 

public IndexShard postRecovery(String reason) throws IndexShardStartedException, IndexShardRelocatedException,
        IndexShardClosedException {
    indicesLifecycle.beforeIndexShardPostRecovery(this);
    synchronized (mutex) {
        if (state == IndexShardState.CLOSED) {
            throw new IndexShardClosedException(shardId);
        }
        if (state == IndexShardState.STARTED) {
            throw new IndexShardStartedException(shardId);
        }
        if (state == IndexShardState.RELOCATED) {
            throw new IndexShardRelocatedException(shardId);
        }
        recoveryState.setStage(RecoveryState.Stage.DONE);
        changeState(IndexShardState.POST_RECOVERY, reason);
    }
    indicesLifecycle.afterIndexShardPostRecovery(this);
    return this;
}
 

IndexShardState recovering(String reason, RecoveryState recoveryState) throws IndexShardStartedException,
        IndexShardRelocatedException, IndexShardRecoveringException, IndexShardClosedException {

    synchronized (mutex) {
        if (state == IndexShardState.CLOSED) {
            throw new IndexShardClosedException(shardId);
        }
        if (state == IndexShardState.STARTED) {
            throw new IndexShardStartedException(shardId);
        }
        if (state == IndexShardState.RELOCATED) {
            throw new IndexShardRelocatedException(shardId);
        }
        if (state == IndexShardState.RECOVERING) {
            throw new IndexShardRecoveringException(shardId);
        }
        if (state == IndexShardState.POST_RECOVERY) {
            throw new IndexShardRecoveringException(shardId);
        }
        this.recoveryState = recoveryState;
        return changeState(IndexShardState.RECOVERING, reason);
    }
}
 

private void addChildImplementations(final RecoveryState recoveryState) {
    childImplementations.put(TOTAL_TIME, new SimpleObjectExpression<Long>() {
        @Override
        public Long value() {
            return recoveryState.getTimer().time();
        }
    });
    childImplementations.put(STAGE, new SimpleObjectExpression<BytesRef>() {
        @Override
        public BytesRef value() {
            return BytesRefs.toBytesRef(recoveryState.getStage().name());
        }
    });
    childImplementations.put(TYPE, new SimpleObjectExpression<BytesRef>() {
        @Override
        public BytesRef value() {
            return BytesRefs.toBytesRef(recoveryState.getType().name());
        }
    });
    childImplementations.put(SIZE, new ShardRecoverySizeExpression(recoveryState));
    childImplementations.put(FILES, new ShardRecoveryFilesExpression(recoveryState));
}
 

private void addChildImplementations(final RecoveryState recoveryState) {
    childImplementations.put(USED, new SimpleObjectExpression<Integer>() {
        @Override
        public Integer value() {
            return recoveryState.getIndex().totalFileCount();
        }
    });
    childImplementations.put(REUSED, new SimpleObjectExpression<Integer>() {
        @Override
        public Integer value() {
            return recoveryState.getIndex().reusedFileCount();
        }
    });
    childImplementations.put(RECOVERED, new SimpleObjectExpression<Integer>() {
        @Override
        public Integer value() {
            return recoveryState.getIndex().recoveredFileCount();
        }
    });
    childImplementations.put(PERCENT, new SimpleObjectExpression<Float>() {
        @Override
        public Float value() {
            return recoveryState.getIndex().recoveredFilesPercent();
        }
    });
}
 

public RecoveryResponse(StreamInput in) throws IOException {
    super(in);
    int size = in.readVInt();
    shardRecoveryStates = new HashMap<>();
    for (int i = 0; i < size; i++) {
        String s = in.readString();
        int listSize = in.readVInt();
        List<RecoveryState> list = new ArrayList<>(listSize);
        for (int j = 0; j < listSize; j++) {
            list.add(new RecoveryState(in));
        }
        shardRecoveryStates.put(s, list);
    }
}
 

/**
 * opens the engine on top of the existing lucene engine and translog.
 * Operations from the translog will be replayed to bring lucene up to date.
 **/
public void openEngineAndRecoverFromTranslog() throws IOException {
    final RecoveryState.Translog translogRecoveryStats = recoveryState.getTranslog();
    final Engine.TranslogRecoveryRunner translogRecoveryRunner = (engine, snapshot) -> {
        translogRecoveryStats.totalOperations(snapshot.totalOperations());
        translogRecoveryStats.totalOperationsOnStart(snapshot.totalOperations());
        return runTranslogRecovery(engine, snapshot, Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY,
            translogRecoveryStats::incrementRecoveredOperations);
    };
    innerOpenEngineAndTranslog();
    getEngine().recoverFromTranslog(translogRecoveryRunner, Long.MAX_VALUE);
}
 

/**
 * perform the last stages of recovery once all translog operations are done.
 * note that you should still call {@link #postRecovery(String)}.
 */
public void finalizeRecovery() {
    recoveryState().setStage(RecoveryState.Stage.FINALIZE);
    Engine engine = getEngine();
    engine.refresh("recovery_finalization");
    engine.config().setEnableGcDeletes(true);
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int size = in.readVInt();
    for (int i = 0; i < size; i++) {
        String s = in.readString();
        int listSize = in.readVInt();
        List<RecoveryState> list = new ArrayList<>(listSize);
        for (int j = 0; j < listSize; j++) {
            list.add(RecoveryState.readRecoveryState(in));
        }
        shardRecoveryStates.put(s, list);
    }
}
 

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    out.writeVInt(shardRecoveryStates.size());
    for (Map.Entry<String, List<RecoveryState>> entry : shardRecoveryStates.entrySet()) {
        out.writeString(entry.getKey());
        out.writeVInt(entry.getValue().size());
        for (RecoveryState recoveryState : entry.getValue()) {
            recoveryState.writeTo(out);
        }
    }
}
 

protected void recoverShardFromStore(IndexShard primary) throws IOException {
    primary.markAsRecovering("store", new RecoveryState(primary.routingEntry(),
        getFakeDiscoNode(primary.routingEntry().currentNodeId()),
        null));
    primary.recoverFromStore();
    updateRoutingEntry(primary, ShardRoutingHelper.moveToStarted(primary.routingEntry()));
}
 

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    out.writeVInt(shardRecoveryStates.size());
    for (Map.Entry<String, List<RecoveryState>> entry : shardRecoveryStates.entrySet()) {
        out.writeString(entry.getKey());
        out.writeVInt(entry.getValue().size());
        for (RecoveryState recoveryState : entry.getValue()) {
            recoveryState.writeTo(out);
        }
    }
}
 

/**
 * Constructs new restore context
 *
 * @param shardId       shard id to restore into
 * @param snapshotId    snapshot id
 * @param recoveryState recovery state to report progress
 * @param bufferSize    buffer size for restore
 */
protected FileRestoreContext(String repositoryName, ShardId shardId, SnapshotId snapshotId, RecoveryState recoveryState,
                             int bufferSize) {
    this.repositoryName = repositoryName;
    this.recoveryState = recoveryState;
    this.snapshotId = snapshotId;
    this.shardId = shardId;
    this.bufferSize = bufferSize;
}
 

/**
 * {@inheritDoc}
 */
@Override
public void restore(SnapshotId snapshotId, Version version, ShardId shardId, ShardId snapshotShardId, RecoveryState recoveryState) {
    final RestoreContext snapshotContext = new RestoreContext(snapshotId, version, shardId, snapshotShardId, recoveryState);
    try {
        snapshotContext.restore();
    } catch (Throwable e) {
        throw new IndexShardRestoreFailedException(shardId, "failed to restore snapshot [" + snapshotId.getSnapshot() + "]", e);
    }
}
 

/**
 * perform the last stages of recovery once all translog operations are done.
 * note that you should still call {@link #postRecovery(String)}.
 */
public void finalizeRecovery() {
    recoveryState().setStage(RecoveryState.Stage.FINALIZE);
    engine().refresh("recovery_finalization");
    startScheduledTasksIfNeeded();

    engineConfig.setEnableGcDeletes(true);
}
 

/**
 * Restores shard from {@link RestoreSource} associated with this shard in routing table
 *
 * @param recoveryState recovery state
 */
private void restore(final IndexShard indexShard, final RecoveryState recoveryState) {
    RestoreSource restoreSource = indexShard.routingEntry().restoreSource();
    if (restoreSource == null) {
        throw new IndexShardRestoreFailedException(shardId, "empty restore source");
    }
    if (logger.isTraceEnabled()) {
        logger.trace("[{}] restoring shard  [{}]", restoreSource.snapshotId(), shardId);
    }
    try {
        recoveryState.getTranslog().totalOperations(0);
        recoveryState.getTranslog().totalOperationsOnStart(0);
        indexShard.prepareForIndexRecovery();
        IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(restoreSource.snapshotId().getRepository());
        ShardId snapshotShardId = shardId;
        if (!shardId.getIndex().equals(restoreSource.index())) {
            snapshotShardId = new ShardId(restoreSource.index(), shardId.id());
        }
        indexShardRepository.restore(restoreSource.snapshotId(), restoreSource.version(), shardId, snapshotShardId, recoveryState);
        indexShard.skipTranslogRecovery();
        indexShard.finalizeRecovery();
        indexShard.postRecovery("restore done");
        restoreService.indexShardRestoreCompleted(restoreSource.snapshotId(), shardId);
    } catch (Throwable t) {
        if (Lucene.isCorruptionException(t)) {
            restoreService.failRestore(restoreSource.snapshotId(), shardId());
        }
        throw new IndexShardRestoreFailedException(shardId, "restore failed", t);
    }
}
 

/**
 * called if recovery has to be restarted after network error / delay **
 */
public void performRecoveryRestart() throws IOException {
    synchronized (mutex) {
        if (state != IndexShardState.RECOVERING) {
            throw new IndexShardNotRecoveringException(shardId, state);
        }
        final Engine engine = this.currentEngineReference.getAndSet(null);
        IOUtils.close(engine);
        recoveryState().setStage(RecoveryState.Stage.INIT);
    }
}
 

private Map<String, Mapping> internalPerformTranslogRecovery(boolean skipTranslogRecovery, boolean indexExists) {
    if (state != IndexShardState.RECOVERING) {
        throw new IndexShardNotRecoveringException(shardId, state);
    }

    recoveryState.setStage(RecoveryState.Stage.VERIFY_INDEX);
    // also check here, before we apply the translog
    if (Booleans.parseBoolean(checkIndexOnStartup, false)) {
        try {
            checkIndex();
        } catch (IOException ex) {
            throw new RecoveryFailedException(recoveryState, "check index failed", ex);
        }
    }
    recoveryState.setStage(RecoveryState.Stage.TRANSLOG);
    // we disable deletes since we allow for operations to be executed against the shard while recovering
    // but we need to make sure we don't loose deletes until we are done recovering
    engineConfig.setEnableGcDeletes(false);
    engineConfig.setCreate(indexExists == false);
    if (recoveryState.getType() == RecoveryState.Type.SNAPSHOT) {
        engineConfig.setIgnoreTranslogStatus(true);
    } else {
        engineConfig.setIgnoreTranslogStatus(false);
    }
    if (skipTranslogRecovery == false) {
        // This will activate our shard so we get our fair share of the indexing buffer during recovery:
        markLastWrite();
    }
    createNewEngine(skipTranslogRecovery, engineConfig);
    return engineConfig.getTranslogRecoveryPerformer().getRecoveredTypes();
}
 

private void addRecoveredFileDetails(SegmentInfos si, Store store, RecoveryState.Index index) throws IOException {
    final Directory directory = store.directory();
    for (String name : Lucene.files(si)) {
        long length = directory.fileLength(name);
        index.addFileDetail(name, length, true);
    }
}
 

void addIndices(final RecoveryState.Index indexRecoveryStats,
                final Directory target,
                final Directory[] sources,
                final long maxSeqNo,
                final long maxUnsafeAutoIdTimestamp,
                IndexMetaData indexMetaData,
                int shardId,
                boolean split) throws IOException {

    // clean target directory (if previous recovery attempt failed) and create a fresh segment file with the proper lucene version
    Lucene.cleanLuceneIndex(target);
    assert sources.length > 0;
    final int luceneIndexCreatedVersionMajor = Lucene.readSegmentInfos(sources[0]).getIndexCreatedVersionMajor();
    new SegmentInfos(luceneIndexCreatedVersionMajor).commit(target);

    final Directory hardLinkOrCopyTarget = new org.apache.lucene.store.HardlinkCopyDirectoryWrapper(target);

    IndexWriterConfig iwc = new IndexWriterConfig(null)
        .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
        .setCommitOnClose(false)
        // we don't want merges to happen here - we call maybe merge on the engine
        // later once we stared it up otherwise we would need to wait for it here
        // we also don't specify a codec here and merges should use the engines for this index
        .setMergePolicy(NoMergePolicy.INSTANCE)
        .setOpenMode(IndexWriterConfig.OpenMode.APPEND);

    try (IndexWriter writer = new IndexWriter(new StatsDirectoryWrapper(hardLinkOrCopyTarget, indexRecoveryStats), iwc)) {
        writer.addIndexes(sources);
        if (split) {
            writer.deleteDocuments(new ShardSplittingQuery(indexMetaData, shardId));
        }
        /*
         * We set the maximum sequence number and the local checkpoint on the target to the maximum of the maximum sequence numbers on
         * the source shards. This ensures that history after this maximum sequence number can advance and we have correct
         * document-level semantics.
         */
        writer.setLiveCommitData(() -> {
            final HashMap<String, String> liveCommitData = new HashMap<>(3);
            liveCommitData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(maxSeqNo));
            liveCommitData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(maxSeqNo));
            liveCommitData.put(Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, Long.toString(maxUnsafeAutoIdTimestamp));
            return liveCommitData.entrySet().iterator();
        });
        writer.commit();
    }
}
 

StatsDirectoryWrapper(Directory in, RecoveryState.Index indexRecoveryStats) {
    super(in);
    this.index = indexRecoveryStats;
}
 

private IndexShard mockIndexShard() {
    IndexService indexService = mock(IndexService.class);
    indexUUID = UUIDs.randomBase64UUID();
    Index index = new Index(indexName, indexUUID);
    ShardId shardId = new ShardId(indexName, indexUUID, 1);

    IndexShard indexShard = mock(IndexShard.class);
    when(indexService.index()).thenReturn(index);
    when(indexShard.shardId()).thenReturn(shardId);
    when(indexShard.state()).thenReturn(IndexShardState.STARTED);

    StoreStats storeStats = new StoreStats(123456L);
    when(indexShard.storeStats()).thenReturn(storeStats);

    Path dataPath = Paths.get("/dummy/" + indexUUID + "/" + shardId.id());
    when(indexShard.shardPath()).thenReturn(new ShardPath(false, dataPath, dataPath, shardId));

    DocsStats docsStats = new DocsStats(654321L, 0L, 200L);
    when(indexShard.docStats()).thenReturn(docsStats).thenThrow(IllegalIndexShardStateException.class);

    ShardRouting shardRouting = ShardRouting.newUnassigned(
        shardId, true, RecoverySource.PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    shardRouting = ShardRoutingHelper.initialize(shardRouting, "node1");
    shardRouting = ShardRoutingHelper.moveToStarted(shardRouting);
    shardRouting = ShardRoutingHelper.relocate(shardRouting, "node_X");
    when(indexShard.routingEntry()).thenReturn(shardRouting);
    when(indexShard.minimumCompatibleVersion()).thenReturn(Version.LATEST);

    RecoveryState recoveryState = mock(RecoveryState.class);
    when(indexShard.recoveryState()).thenReturn(recoveryState);
    RecoveryState.Index recoveryStateIndex = mock(RecoveryState.Index.class);
    RecoveryState.Timer recoveryStateTimer = mock(RecoveryState.Timer.class);
    when(recoveryState.getRecoverySource()).thenReturn(RecoverySource.PeerRecoverySource.INSTANCE);
    when(recoveryState.getIndex()).thenReturn(recoveryStateIndex);
    when(recoveryState.getStage()).thenReturn(RecoveryState.Stage.DONE);
    when(recoveryState.getTimer()).thenReturn(recoveryStateTimer);

    when(recoveryStateIndex.totalBytes()).thenReturn(2048L);
    when(recoveryStateIndex.reusedBytes()).thenReturn(1024L);
    when(recoveryStateIndex.recoveredBytes()).thenReturn(1024L);
    when(recoveryStateIndex.totalFileCount()).thenReturn(2);
    when(recoveryStateIndex.reusedFileCount()).thenReturn(1);
    when(recoveryStateIndex.recoveredFileCount()).thenReturn(1);
    when(recoveryStateTimer.time()).thenReturn(10000L);

    return indexShard;
}
 

@Override
protected RecoveryState shardOperation(RecoveryRequest request, ShardRouting shardRouting) {
    IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex());
    IndexShard indexShard = indexService.getShard(shardRouting.shardId().id());
    return indexShard.recoveryState();
}