org.elasticsearch.index.seqno.SequenceNumbers Java Examples

private static LocalCheckpointTracker createLocalCheckpointTracker(EngineConfig engineConfig, SegmentInfos lastCommittedSegmentInfos,
    Logger logger, Supplier<Searcher> searcherSupplier, BiFunction<Long, Long, LocalCheckpointTracker> localCheckpointTrackerSupplier) {
    try {
        final SequenceNumbers.CommitInfo seqNoStats =
            SequenceNumbers.loadSeqNoInfoFromLuceneCommit(lastCommittedSegmentInfos.userData.entrySet());
        final long maxSeqNo = seqNoStats.maxSeqNo;
        final long localCheckpoint = seqNoStats.localCheckpoint;
        logger.trace("recovered maximum sequence number [{}] and local checkpoint [{}]", maxSeqNo, localCheckpoint);
        final LocalCheckpointTracker tracker = localCheckpointTrackerSupplier.apply(maxSeqNo, localCheckpoint);
        // Operations that are optimized using max_seq_no_of_updates optimization must not be processed twice; otherwise, they will
        // create duplicates in Lucene. To avoid this we check the LocalCheckpointTracker to see if an operation was already processed.
        // Thus, we need to restore the LocalCheckpointTracker bit by bit to ensure the consistency between LocalCheckpointTracker and
        // Lucene index. This is not the only solution since we can bootstrap max_seq_no_of_updates with max_seq_no of the commit to
        // disable the MSU optimization during recovery. Here we prefer to maintain the consistency of LocalCheckpointTracker.
        if (localCheckpoint < maxSeqNo && engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
            try (Searcher searcher = searcherSupplier.get()) {
                Lucene.scanSeqNosInReader(searcher.getDirectoryReader(), localCheckpoint + 1, maxSeqNo,
                    tracker::markSeqNoAsCompleted);
            }
        }
        return tracker;
    } catch (IOException ex) {
        throw new EngineCreationFailureException(engineConfig.getShardId(), "failed to create local checkpoint tracker", ex);
    }
}
 

@Override
public boolean hasCompleteOperationHistory(String source, MapperService mapperService, long startingSeqNo) throws IOException {
    if (engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
        return getMinRetainedSeqNo() <= startingSeqNo;
    } else {
        final long currentLocalCheckpoint = getLocalCheckpointTracker().getProcessedCheckpoint();
        final LocalCheckpointTracker tracker = new LocalCheckpointTracker(startingSeqNo, startingSeqNo - 1);
        try (Translog.Snapshot snapshot = getTranslog().newSnapshotFromMinSeqNo(startingSeqNo)) {
            Translog.Operation operation;
            while ((operation = snapshot.next()) != null) {
                if (operation.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
                    tracker.markSeqNoAsProcessed(operation.seqNo());
                }
            }
        }
        return tracker.getProcessedCheckpoint() >= currentLocalCheckpoint;
    }
}
 

public static Translog.Location performOnReplica(ResyncReplicationRequest request, IndexShard replica) throws Exception {
    Translog.Location location = null;
    /*
     * Operations received from resync do not have auto_id_timestamp individually, we need to bootstrap this max_seen_timestamp
     * (at least the highest timestamp from any of these operations) to make sure that we will disable optimization for the same
     * append-only requests with timestamp (sources of these operations) that are replicated; otherwise we may have duplicates.
     */
    replica.updateMaxUnsafeAutoIdTimestamp(request.getMaxSeenAutoIdTimestampOnPrimary());
    for (Translog.Operation operation : request.getOperations()) {
        final Engine.Result operationResult = replica.applyTranslogOperation(operation, Engine.Operation.Origin.REPLICA);
        if (operationResult.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
            throw new TransportReplicationAction.RetryOnReplicaException(replica.shardId(),
                "Mappings are not available on the replica yet, triggered update: " + operationResult.getRequiredMappingUpdate());
        }
        location = syncOperationResultOrThrow(operationResult, location);
    }
    if (request.getTrimAboveSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        replica.trimOperationOfPreviousPrimaryTerms(request.getTrimAboveSeqNo());
    }
    return location;
}
 

/**
 * Find the highest index position of a safe index commit whose max sequence number is not greater than the global checkpoint.
 * Index commits with different translog UUID will be filtered out as they don't belong to this engine.
 */
private static int indexOfKeptCommits(List<? extends IndexCommit> commits, long globalCheckpoint) throws IOException {
    final String expectedTranslogUUID = commits.get(commits.size() - 1).getUserData().get(Translog.TRANSLOG_UUID_KEY);

    // Commits are sorted by age (the 0th one is the oldest commit).
    for (int i = commits.size() - 1; i >= 0; i--) {
        final Map<String, String> commitUserData = commits.get(i).getUserData();
        // Ignore index commits with different translog uuid.
        if (expectedTranslogUUID.equals(commitUserData.get(Translog.TRANSLOG_UUID_KEY)) == false) {
            return i + 1;
        }
        final long maxSeqNoFromCommit = Long.parseLong(commitUserData.get(SequenceNumbers.MAX_SEQ_NO));
        if (maxSeqNoFromCommit <= globalCheckpoint) {
            return i;
        }
    }
    // If an index was created before 6.2 or recovered from remote, we might not have a safe commit.
    // In this case, we return the oldest index commit instead.
    return 0;
}
 

protected IndexRequestBuilder createUpdateRequest(final EsAbstractEntity esEntity) {
    final IndexRequestBuilder builder =
            client.prepareIndex().setIndex(asEsIndex()).setId(esEntity.asDocMeta().id()).setSource(toSource(esEntity));
    final RequestOptionCall<IndexRequestBuilder> indexOption = esEntity.asDocMeta().indexOption();
    if (indexOption != null) {
        indexOption.callback(builder);
    }
    final Long seqNo = esEntity.asDocMeta().seqNo();
    if (seqNo != null && seqNo.longValue() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        esEntity.asDocMeta().seqNo(seqNo);
    }
    final Long primaryTerm = esEntity.asDocMeta().primaryTerm();
    if (primaryTerm != null && primaryTerm.longValue() != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) {
        esEntity.asDocMeta().primaryTerm(primaryTerm);
    }
    return builder;
}
 

@Override
protected int delegateUpdate(final Entity entity, final UpdateOption<? extends ConditionBean> option) {
    final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
    final IndexRequestBuilder builder = createUpdateRequest(esEntity);

    final IndexResponse response = builder.execute().actionGet(indexTimeout);
    final long seqNo = response.getSeqNo();
    if (seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        esEntity.asDocMeta().seqNo(seqNo);
    }
    final long primaryTerm = response.getPrimaryTerm();
    if (primaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) {
        esEntity.asDocMeta().primaryTerm(primaryTerm);
    }

    return 1;
}
 

protected IndexRequestBuilder createUpdateRequest(final EsAbstractEntity esEntity) {
    final IndexRequestBuilder builder =
            client.prepareIndex().setIndex(asEsIndex()).setId(esEntity.asDocMeta().id()).setSource(toSource(esEntity));
    final RequestOptionCall<IndexRequestBuilder> indexOption = esEntity.asDocMeta().indexOption();
    if (indexOption != null) {
        indexOption.callback(builder);
    }
    final Long seqNo = esEntity.asDocMeta().seqNo();
    if (seqNo != null && seqNo.longValue() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        esEntity.asDocMeta().seqNo(seqNo);
    }
    final Long primaryTerm = esEntity.asDocMeta().primaryTerm();
    if (primaryTerm != null && primaryTerm.longValue() != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) {
        esEntity.asDocMeta().primaryTerm(primaryTerm);
    }
    return builder;
}
 

@Override
protected int delegateUpdate(final Entity entity, final UpdateOption<? extends ConditionBean> option) {
    final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
    final IndexRequestBuilder builder = createUpdateRequest(esEntity);

    final IndexResponse response = builder.execute().actionGet(indexTimeout);
    final long seqNo = response.getSeqNo();
    if (seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        esEntity.asDocMeta().seqNo(seqNo);
    }
    final long primaryTerm = response.getPrimaryTerm();
    if (primaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) {
        esEntity.asDocMeta().primaryTerm(primaryTerm);
    }

    return 1;
}
 

@Override
protected int delegateUpdate(final Entity entity, final UpdateOption<? extends ConditionBean> option) {
    final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
    final IndexRequestBuilder builder = createUpdateRequest(esEntity);

    final IndexResponse response = builder.execute().actionGet(indexTimeout);
    final long seqNo = response.getSeqNo();
    if (seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        esEntity.asDocMeta().seqNo(seqNo);
    }
    final long primaryTerm = response.getPrimaryTerm();
    if (primaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) {
        esEntity.asDocMeta().primaryTerm(primaryTerm);
    }

    return 1;
}
 

protected InternalEngine createEngine(@Nullable IndexWriterFactory indexWriterFactory,
                                      @Nullable BiFunction<Long, Long, LocalCheckpointTracker> localCheckpointTrackerSupplier,
                                      @Nullable ToLongBiFunction<Engine, Engine.Operation> seqNoForOperation,
                                      EngineConfig config) throws IOException {
    final Store store = config.getStore();
    final Directory directory = store.directory();
    if (Lucene.indexExists(directory) == false) {
        store.createEmpty();
        final String translogUuid = Translog.createEmptyTranslog(config.getTranslogConfig().getTranslogPath(),
            SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get());
        store.associateIndexWithNewTranslog(translogUuid);

    }
    InternalEngine internalEngine = createInternalEngine(indexWriterFactory, localCheckpointTrackerSupplier, seqNoForOperation, config);
    internalEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
    return internalEngine;
}
 

protected IndexRequestBuilder createUpdateRequest(final EsAbstractEntity esEntity) {
    final IndexRequestBuilder builder =
            client.prepareIndex().setIndex(asEsIndex()).setId(esEntity.asDocMeta().id()).setSource(toSource(esEntity));
    final RequestOptionCall<IndexRequestBuilder> indexOption = esEntity.asDocMeta().indexOption();
    if (indexOption != null) {
        indexOption.callback(builder);
    }
    final Long seqNo = esEntity.asDocMeta().seqNo();
    if (seqNo != null && seqNo.longValue() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        esEntity.asDocMeta().seqNo(seqNo);
    }
    final Long primaryTerm = esEntity.asDocMeta().primaryTerm();
    if (primaryTerm != null && primaryTerm.longValue() != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) {
        esEntity.asDocMeta().primaryTerm(primaryTerm);
    }
    return builder;
}
 

static Checkpoint emptyTranslogCheckpoint(final long offset, final long generation, final long globalCheckpoint,
                                          long minTranslogGeneration) {
    final long minSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    final long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    final long trimmedAboveSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
    return new Checkpoint(offset, 0, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo);
}
 

static Checkpoint readCheckpointV6_0_0(final DataInput in) throws IOException {
    final long offset = in.readLong();
    final int numOps = in.readInt();
    final long generation = in.readLong();
    final long minSeqNo = in.readLong();
    final long maxSeqNo = in.readLong();
    final long globalCheckpoint = in.readLong();
    final long minTranslogGeneration = in.readLong();
    final long trimmedAboveSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
    return new Checkpoint(offset, numOps, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo);
}
 

static Checkpoint readCheckpointV5_0_0(final DataInput in) throws IOException {
    final long offset = in.readLong();
    final int numOps = in.readInt();
    final long generation = in.readLong();
    final long minSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    final long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    final long globalCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
    final long minTranslogGeneration = -1;
    final long trimmedAboveSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
    return new Checkpoint(offset, numOps, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo);
}
 

/**
 * Closes current reader and creates new one with new checkoint and same file channel
 */
TranslogReader closeIntoTrimmedReader(long aboveSeqNo, ChannelFactory channelFactory) throws IOException {
    if (closed.compareAndSet(false, true)) {
        Closeable toCloseOnFailure = channel;
        final TranslogReader newReader;
        try {
            if (aboveSeqNo < checkpoint.trimmedAboveSeqNo
                || aboveSeqNo < checkpoint.maxSeqNo && checkpoint.trimmedAboveSeqNo == SequenceNumbers.UNASSIGNED_SEQ_NO) {
                final Path checkpointFile = path.getParent().resolve(getCommitCheckpointFileName(checkpoint.generation));
                final Checkpoint newCheckpoint = new Checkpoint(checkpoint.offset, checkpoint.numOps,
                    checkpoint.generation, checkpoint.minSeqNo, checkpoint.maxSeqNo,
                    checkpoint.globalCheckpoint, checkpoint.minTranslogGeneration, aboveSeqNo);
                Checkpoint.write(channelFactory, checkpointFile, newCheckpoint, StandardOpenOption.WRITE);

                IOUtils.fsync(checkpointFile, false);
                IOUtils.fsync(checkpointFile.getParent(), true);

                newReader = new TranslogReader(newCheckpoint, channel, path, header);
            } else {
                newReader = new TranslogReader(checkpoint, channel, path, header);
            }
            toCloseOnFailure = null;
            return newReader;
        } finally {
            IOUtils.close(toCloseOnFailure);
        }
    } else {
        throw new AlreadyClosedException(toString() + " is already closed");
    }
}
 

private SoftDeletesPolicy newSoftDeletesPolicy() throws IOException {
    final Map<String, String> commitUserData = store.readLastCommittedSegmentsInfo().userData;
    final long lastMinRetainedSeqNo;
    if (commitUserData.containsKey(Engine.MIN_RETAINED_SEQNO)) {
        lastMinRetainedSeqNo = Long.parseLong(commitUserData.get(Engine.MIN_RETAINED_SEQNO));
    } else {
        lastMinRetainedSeqNo = Long.parseLong(commitUserData.get(SequenceNumbers.MAX_SEQ_NO)) + 1;
    }
    return new SoftDeletesPolicy(
            translog::getLastSyncedGlobalCheckpoint,
            lastMinRetainedSeqNo,
            engineConfig.getIndexSettings().getSoftDeleteRetentionOperations(),
            engineConfig.retentionLeasesSupplier());
}
 

/**
 * creates an empty lucene index and a corresponding empty translog. Any existing data will be deleted.
 */
public void createEmpty() throws IOException {
    metadataLock.writeLock().lock();
    try (IndexWriter writer = newIndexWriter(IndexWriterConfig.OpenMode.CREATE, directory, null)) {
        final Map<String, String> map = new HashMap<>();
        map.put(Engine.HISTORY_UUID_KEY, UUIDs.randomBase64UUID());
        map.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(SequenceNumbers.NO_OPS_PERFORMED));
        map.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(SequenceNumbers.NO_OPS_PERFORMED));
        map.put(Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, "-1");
        updateCommitData(writer, map);
    } finally {
        metadataLock.writeLock().unlock();
    }
}
 

/**
 * Marks an existing lucene index with a new history uuid.
 * This is used to make sure no existing shard will recovery from this index using ops based recovery.
 */
public void bootstrapNewHistory() throws IOException {
    metadataLock.writeLock().lock();
    try {
        Map<String, String> userData = readLastCommittedSegmentsInfo().getUserData();
        final SequenceNumbers.CommitInfo seqno = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(userData.entrySet());
        bootstrapNewHistory(seqno.maxSeqNo);
    } finally {
        metadataLock.writeLock().unlock();
    }
}
 

/**
 * Marks an existing lucene index with a new history uuid and sets the given maxSeqNo as the local checkpoint
 * as well as the maximum sequence number.
 * This is used to make sure no existing shard will recover from this index using ops based recovery.
 * @see SequenceNumbers#LOCAL_CHECKPOINT_KEY
 * @see SequenceNumbers#MAX_SEQ_NO
 */
public void bootstrapNewHistory(long maxSeqNo) throws IOException {
    metadataLock.writeLock().lock();
    try (IndexWriter writer = newIndexWriter(IndexWriterConfig.OpenMode.APPEND, directory, null)) {
        final Map<String, String> userData = getUserData(writer);
        final Map<String, String> map = new HashMap<>();
        map.put(Engine.HISTORY_UUID_KEY, UUIDs.randomBase64UUID());
        map.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(maxSeqNo));
        map.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(maxSeqNo));
        logger.debug("bootstrap a new history_uuid [{}], user_data [{}]", map, userData);
        updateCommitData(writer, map);
    } finally {
        metadataLock.writeLock().unlock();
    }
}
 

static long maxSeqNosInReader(DirectoryReader reader) throws IOException {
    long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    for (LeafReaderContext leaf : reader.leaves()) {
        final NumericDocValues seqNoDocValues = leaf.reader().getNumericDocValues(SeqNoFieldMapper.NAME);
        while (seqNoDocValues.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
            maxSeqNo = SequenceNumbers.max(maxSeqNo, seqNoDocValues.longValue());
        }
    }
    return maxSeqNo;
}
 

@Override
protected WriteReplicaResult<ShardDeleteRequest> processRequestItemsOnReplica(IndexShard indexShard, ShardDeleteRequest request) throws IOException {
    Translog.Location translogLocation = null;
    for (ShardDeleteRequest.Item item : request.items()) {
        int location = item.location();
        if (request.skipFromLocation() == location) {
            // skipping this and all next items, the primary did not processed them (mostly due to a kill request)
            break;
        }

        // Only execute delete operation on replica if the sequence number was applied from primary.
        // If that's not the case, the delete on primary didn't succeed. Note that we still need to
        // process the other items in case of a bulk request.
        if (item.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
            Engine.DeleteResult deleteResult = indexShard.applyDeleteOperationOnReplica(item.seqNo(),
                                                                                        item.version(),
                                                                                        Constants.DEFAULT_MAPPING_TYPE,
                                                                                        item.id());

            translogLocation = deleteResult.getTranslogLocation();
            if (logger.isTraceEnabled()) {
                logger.trace("shardId={} REPLICA: successfully deleted id={}", request.shardId(), item.id());
            }
        }
    }
    return new WriteReplicaResult<>(request, translogLocation, null, indexShard, logger);
}
 

public StartRecoveryRequest getStartRecoveryRequest() throws IOException {
    Store.MetadataSnapshot metadataSnapshot = randomBoolean() ? Store.MetadataSnapshot.EMPTY :
        new Store.MetadataSnapshot(Collections.emptyMap(),
                                   Collections.singletonMap(Engine.HISTORY_UUID_KEY, UUIDs.randomBase64UUID()), randomIntBetween(0, 100));
    return new StartRecoveryRequest(
        shardId,
        null,
        new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
        new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
        metadataSnapshot,
        randomBoolean(),
        randomNonNegativeLong(),
        randomBoolean() || metadataSnapshot.getHistoryUUID() == null ?
            SequenceNumbers.UNASSIGNED_SEQ_NO : randomNonNegativeLong());
}
 

@Test
public void testSendSnapshotStopOnError() throws Exception {
    final int fileChunkSizeInBytes = between(1, 10 * 1024);
    final StartRecoveryRequest request = getStartRecoveryRequest();
    final IndexShard shard = mock(IndexShard.class);
    when(shard.state()).thenReturn(IndexShardState.STARTED);
    final List<Translog.Operation> ops = new ArrayList<>();
    for (int numOps = between(1, 256), i = 0; i < numOps; i++) {
        final Engine.Index index = getIndex(Integer.toString(i));
        ops.add(new Translog.Index(index, new Engine.IndexResult(1, 1, i, true)));
    }
    final AtomicBoolean wasFailed = new AtomicBoolean();
    RecoveryTargetHandler recoveryTarget = new TestRecoveryTargetHandler() {
        @Override
        public void indexTranslogOperations(List<Translog.Operation> operations, int totalTranslogOps, long timestamp,
                                            long msu, ActionListener<Long> listener) {
            if (randomBoolean()) {
                maybeExecuteAsync(() -> listener.onResponse(SequenceNumbers.NO_OPS_PERFORMED));
            } else {
                maybeExecuteAsync(() -> listener.onFailure(new RuntimeException("test - failed to index")));
                wasFailed.set(true);
            }
        }
    };
    RecoverySourceHandler handler = new RecoverySourceHandler(shard, recoveryTarget, request, fileChunkSizeInBytes, between(1, 10));
    PlainActionFuture<RecoverySourceHandler.SendSnapshotResult> future = new PlainActionFuture<>();
    final long startingSeqNo = randomLongBetween(0, ops.size() - 1L);
    final long endingSeqNo = randomLongBetween(startingSeqNo, ops.size() - 1L);
    handler.phase2(startingSeqNo, startingSeqNo, endingSeqNo, newTranslogSnapshot(ops, Collections.emptyList()),
                   randomNonNegativeLong(), randomNonNegativeLong(), future);
    if (wasFailed.get()) {
        assertThat(expectThrows(RuntimeException.class, future::actionGet).getMessage(), equalTo("test - failed to index"));
    }
}
 

@Test
public void testVerifySeqNoStatsWhenRecoverWithSyncId() throws Exception {
    IndexShard shard = mock(IndexShard.class);
    when(shard.state()).thenReturn(IndexShardState.STARTED);
    RecoverySourceHandler handler = new RecoverySourceHandler(
        shard, new TestRecoveryTargetHandler(), getStartRecoveryRequest(), between(1, 16), between(1, 4));

    String syncId = UUIDs.randomBase64UUID();
    int numDocs = between(0, 1000);
    long localCheckpoint = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE);
    long maxSeqNo = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE);
    assertTrue(handler.canSkipPhase1(
        newMetadataSnapshot(syncId, Long.toString(localCheckpoint), Long.toString(maxSeqNo), numDocs),
        newMetadataSnapshot(syncId, Long.toString(localCheckpoint), Long.toString(maxSeqNo), numDocs)));

    Class<? extends Throwable> expectedError = Assertions.ENABLED ? AssertionError.class : IllegalStateException.class;
    Throwable error = expectThrows(expectedError, () -> {
        long localCheckpointOnTarget = randomValueOtherThan(
            localCheckpoint,
            () -> randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE));
        long maxSeqNoOnTarget = randomValueOtherThan(
            maxSeqNo,
            () -> randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE));
        handler.canSkipPhase1(
            newMetadataSnapshot(syncId, Long.toString(localCheckpoint), Long.toString(maxSeqNo), numDocs),
            newMetadataSnapshot(syncId, Long.toString(localCheckpointOnTarget), Long.toString(maxSeqNoOnTarget), numDocs));
    });
    assertThat(error.getMessage(), containsString("try to recover [index][1] with sync id but seq_no stats are mismatched:"));
}
 

private Store.MetadataSnapshot newMetadataSnapshot(String syncId, String localCheckpoint, String maxSeqNo, int numDocs) {
    HashMap<String, String> userData = new HashMap<>();
    userData.put(Engine.SYNC_COMMIT_ID, syncId);
    if (localCheckpoint != null) {
        userData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, localCheckpoint);
    }
    if (maxSeqNo != null) {
        userData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, maxSeqNo);
    }
    return new Store.MetadataSnapshot(Collections.emptyMap(), userData, numDocs);
}
 

/**
 * Asserts that the max_seq_no stored in the commit's user_data is never smaller than seq_no of any document in the commit.
 */
public static void assertMaxSeqNoInCommitUserData(Engine engine) throws Exception {
    List<IndexCommit> commits = DirectoryReader.listCommits(engine.store.directory());
    for (IndexCommit commit : commits) {
        try (DirectoryReader reader = DirectoryReader.open(commit)) {
            assertThat(Long.parseLong(commit.getUserData().get(SequenceNumbers.MAX_SEQ_NO)),
                greaterThanOrEqualTo(maxSeqNosInReader(reader)));
        }
    }
}
 

@Test
public void testDuplicateRecoveries() throws IOException {
    IndexShard primary = newStartedShard(true);
    PeerRecoverySourceService peerRecoverySourceService = new PeerRecoverySourceService(
        mock(TransportService.class), mock(IndicesService.class),
        new RecoverySettings(
            Settings.EMPTY,
            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
    StartRecoveryRequest startRecoveryRequest = new StartRecoveryRequest(
        primary.shardId(),
        randomAlphaOfLength(10),
        getFakeDiscoNode("source"),
        getFakeDiscoNode("target"),
        Store.MetadataSnapshot.EMPTY,
        randomBoolean(),
        randomLong(),
        SequenceNumbers.UNASSIGNED_SEQ_NO);
    RecoverySourceHandler handler = peerRecoverySourceService.ongoingRecoveries
        .addNewRecovery(startRecoveryRequest, primary);
    DelayRecoveryException delayRecoveryException = expectThrows(
        DelayRecoveryException.class,
        () -> peerRecoverySourceService.ongoingRecoveries.addNewRecovery(
            startRecoveryRequest,
            primary));
    assertThat(delayRecoveryException.getMessage(), containsString("recovery with same target already registered"));
    peerRecoverySourceService.ongoingRecoveries.remove(primary, handler);
    // re-adding after removing previous attempt works
    handler = peerRecoverySourceService.ongoingRecoveries.addNewRecovery(startRecoveryRequest, primary);
    peerRecoverySourceService.ongoingRecoveries.remove(primary, handler);
    closeShards(primary);
}
 

protected Translog createTranslog(Path translogPath, LongSupplier primaryTermSupplier) throws IOException {
    TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE);
    String translogUUID = Translog.createEmptyTranslog(translogPath, SequenceNumbers.NO_OPS_PERFORMED, shardId,
        primaryTermSupplier.getAsLong());
    return new Translog(translogConfig, translogUUID, createTranslogDeletionPolicy(INDEX_SETTINGS),
        () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTermSupplier, seqNo -> {});
}
 

public List<Engine.Operation> generateHistoryOnReplica(int numOps, boolean allowGapInSeqNo, boolean allowDuplicate) throws Exception {
    long seqNo = 0;
    final int maxIdValue = randomInt(numOps * 2);
    final List<Engine.Operation> operations = new ArrayList<>(numOps);
    for (int i = 0; i < numOps; i++) {
        final String id = Integer.toString(randomInt(maxIdValue));
        final Engine.Operation.TYPE opType = randomFrom(Engine.Operation.TYPE.values());
        final long startTime = threadPool.relativeTimeInMillis();
        final int copies = allowDuplicate && rarely() ? between(2, 4) : 1;
        for (int copy = 0; copy < copies; copy++) {
            final ParsedDocument doc = createParsedDoc(id, null);
            switch (opType) {
                case INDEX:
                    operations.add(new Engine.Index(EngineTestCase.newUid(doc), doc, seqNo, primaryTerm.get(),
                                                    i, null, randomFrom(REPLICA, PEER_RECOVERY), startTime, -1, true, SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
                    break;
                case DELETE:
                    operations.add(new Engine.Delete("default", doc.id(), EngineTestCase.newUid(doc), seqNo, primaryTerm.get(),
                                                     i, null, randomFrom(REPLICA, PEER_RECOVERY), startTime, SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
                    break;
                case NO_OP:
                    operations.add(new Engine.NoOp(seqNo, primaryTerm.get(),
                                                   randomFrom(REPLICA, PEER_RECOVERY), startTime, "test-" + i));
                    break;
                default:
                    throw new IllegalStateException("Unknown operation type [" + opType + "]");
            }
        }
        seqNo++;
        if (allowGapInSeqNo && rarely()) {
            seqNo++;
        }
    }
    Randomness.shuffle(operations);
    return operations;
}
 

protected Engine.Index replicaIndexForDoc(ParsedDocument doc,
                                          long version,
                                          long seqNo,
                                          boolean isRetry) {
    return new Engine.Index(
        newUid(doc), doc, seqNo, primaryTerm.get(), version, null,
        Engine.Operation.Origin.REPLICA, System.nanoTime(), Translog.UNSET_AUTO_GENERATED_TIMESTAMP,
        isRetry, SequenceNumbers.UNASSIGNED_SEQ_NO, 0);
}