Java Code Examples for org.elasticsearch.index.shard.IndexShard

The following examples show how to use org.elasticsearch.index.shard.IndexShard. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: crate   Source File: PeerRecoverySourceService.java    License: Apache License 2.0 7 votes vote down vote up
private void recover(StartRecoveryRequest request, ActionListener<RecoveryResponse> listener) throws IOException {
    final IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
    final IndexShard shard = indexService.getShard(request.shardId().id());

    final ShardRouting routingEntry = shard.routingEntry();

    if (routingEntry.primary() == false || routingEntry.active() == false) {
        throw new DelayRecoveryException("source shard [" + routingEntry + "] is not an active primary");
    }

    if (request.isPrimaryRelocation()
        && (
            routingEntry.relocating() == false
            || routingEntry.relocatingNodeId().equals(request.targetNode().getId()) == false)) {
        LOGGER.debug(
            "delaying recovery of {} as source shard is not marked yet as relocating to {}",
            request.shardId(), request.targetNode());
        throw new DelayRecoveryException("source shard is not marked yet as relocating to [" + request.targetNode() + "]");
    }

    RecoverySourceHandler handler = ongoingRecoveries.addNewRecovery(request, shard);
    LOGGER.trace(
        "[{}][{}] starting recovery to {}",
        request.shardId().getIndex().getName(), request.shardId().id(), request.targetNode());
    handler.recoverToTarget(ActionListener.runAfter(listener, () -> ongoingRecoveries.remove(shard, handler)));
}
 
Example 2
Source Project: Elasticsearch   Source File: RecoverySource.java    License: Apache License 2.0 6 votes vote down vote up
synchronized void cancel(IndexShard shard, String reason) {
    final Set<RecoverySourceHandler> shardRecoveryHandlers = ongoingRecoveries.get(shard);
    if (shardRecoveryHandlers != null) {
        final List<Exception> failures = new ArrayList<>();
        for (RecoverySourceHandler handlers : shardRecoveryHandlers) {
            try {
                handlers.cancel(reason);
            } catch (Exception ex) {
                failures.add(ex);
            } finally {
                shard.recoveryStats().decCurrentAsSource();
            }
        }
        ExceptionsHelper.maybeThrowRuntimeAndSuppress(failures);
    }
}
 
Example 3
Source Project: Elasticsearch   Source File: BlobRecoverySourceHandler.java    License: Apache License 2.0 6 votes vote down vote up
public BlobRecoverySourceHandler(final IndexShard shard, final StartRecoveryRequest request, final RecoverySettings recoverySettings,
                                 final TransportService transportService, final ESLogger logger,
                                 BlobTransferTarget blobTransferTarget, BlobIndices blobIndices) {
    super(shard, request, recoverySettings, transportService, logger);
    this.shard = shard;
    this.request = request;
    this.recoverySettings = recoverySettings;
    this.logger = logger;
    this.transportService = transportService;
    this.indexName = this.request.shardId().index().name();
    this.shardId = this.request.shardId().id();

    this.response = new RecoveryResponse();
    if (BlobIndices.isBlobIndex(shard.shardId().getIndex())) {
        blobRecoveryHandler = new BlobRecoveryHandler(
                transportService, recoverySettings, blobTransferTarget, blobIndices, shard, request);
    } else {
        blobRecoveryHandler = null;
    }
}
 
Example 4
Source Project: Elasticsearch   Source File: PercolatorQueriesRegistry.java    License: Apache License 2.0 6 votes vote down vote up
private int loadQueries(IndexShard shard) {
    shard.refresh("percolator_load_queries");
    // NOTE: we acquire the searcher via the engine directly here since this is executed right
    // before the shard is marked as POST_RECOVERY
    try (Engine.Searcher searcher = shard.engine().acquireSearcher("percolator_load_queries")) {
        Query query = new TermQuery(new Term(TypeFieldMapper.NAME, PercolatorService.TYPE_NAME));
        QueriesLoaderCollector queryCollector = new QueriesLoaderCollector(PercolatorQueriesRegistry.this, logger, mapperService, indexFieldDataService);
        IndexSearcher indexSearcher = new IndexSearcher(searcher.reader());
        indexSearcher.setQueryCache(null);
        indexSearcher.search(query, queryCollector);
        Map<BytesRef, Query> queries = queryCollector.queries();
        for (Map.Entry<BytesRef, Query> entry : queries.entrySet()) {
            Query previousQuery = percolateQueries.put(entry.getKey(), entry.getValue());
            shardPercolateService.addedQuery(entry.getKey(), previousQuery, entry.getValue());
        }
        return queries.size();
    } catch (Exception e) {
        throw new PercolatorException(shardId.index(), "failed to load queries from percolator index", e);
    }
}
 
Example 5
Source Project: Elasticsearch   Source File: TransportShardUpsertAction.java    License: Apache License 2.0 6 votes vote down vote up
private Engine.IndexingOperation prepareIndexOnPrimary(IndexShard indexShard,
                                                       long version,
                                                       ShardUpsertRequest request,
                                                       ShardUpsertRequest.Item item) {
    SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, item.source())
            .type(request.type())
            .id(item.id())
            .routing(request.routing());

    if (logger.isTraceEnabled()) {
        logger.trace("[{}] shard operation with opType={} id={} version={}  source={}",
                indexShard.shardId(), item.opType(), item.id(), version, item.source().toUtf8());
    }
    if (item.opType() == IndexRequest.OpType.INDEX) {
        return indexShard.prepareIndexOnPrimary(sourceToParse, version, item.versionType(), request.canHaveDuplicates());
    }
    return indexShard.prepareCreateOnPrimary(
            sourceToParse, version, item.versionType(), request.canHaveDuplicates(), false);
}
 
Example 6
Source Project: crate   Source File: IndicesService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public IndexShard createShard(ShardRouting shardRouting, RecoveryState recoveryState, PeerRecoveryTargetService recoveryTargetService,
                              PeerRecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
                              Consumer<IndexShard.ShardFailure> onShardFailure,
                              Consumer<ShardId> globalCheckpointSyncer) throws IOException {
    ensureChangesAllowed();
    IndexService indexService = indexService(shardRouting.index());
    IndexShard indexShard = indexService.createShard(shardRouting, globalCheckpointSyncer);
    indexShard.addShardFailureCallback(onShardFailure);
    indexShard.startRecovery(recoveryState, recoveryTargetService, recoveryListener, repositoriesService,
        (type, mapping) -> {
            assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS :
                "mapping update consumer only required by local shards recovery";
            client.admin().indices().preparePutMapping()
                .setConcreteIndex(shardRouting.index()) // concrete index - no name clash, it uses uuid
                .setType(type)
                .setSource(mapping.source().string(), XContentType.JSON)
                .get();
        }, this);
    return indexShard;
}
 
Example 7
Source Project: crate   Source File: TransportWriteAction.java    License: Apache License 2.0 6 votes vote down vote up
public WritePrimaryResult(ReplicaRequest request,
                          @Nullable Response finalResponse,
                          @Nullable Location location,
                          @Nullable Exception operationFailure,
                          IndexShard primary) {
    super(request, finalResponse, operationFailure);
    this.location = location;
    this.primary = primary;
    assert location == null || operationFailure == null
            : "expected either failure to be null or translog location to be null, " +
            "but found: [" + location + "] translog location and [" + operationFailure + "] failure";
    if (operationFailure != null) {
        this.finishedAsyncActions = true;
    } else {
        /*
         * We call this before replication because this might wait for a refresh and that can take a while.
         * This way we wait for the refresh in parallel on the primary and on the replica.
         */
        new AsyncAfterWriteAction(primary, location, this).run();
    }
}
 
Example 8
Source Project: Elasticsearch   Source File: RecoverySource.java    License: Apache License 2.0 6 votes vote down vote up
@Inject
public RecoverySource(Settings settings, TransportService transportService, IndicesService indicesService,
                      RecoverySettings recoverySettings, ClusterService clusterService) {
    super(settings);
    this.transportService = transportService;
    this.indicesService = indicesService;
    this.clusterService = clusterService;
    this.indicesService.indicesLifecycle().addListener(new IndicesLifecycle.Listener() {
        @Override
        public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard,
                                           Settings indexSettings) {
            if (indexShard != null) {
                ongoingRecoveries.cancel(indexShard, "shard is closed");
            }
        }
    });

    this.recoverySettings = recoverySettings;

    transportService.registerRequestHandler(Actions.START_RECOVERY, StartRecoveryRequest.class, ThreadPool.Names.GENERIC, new StartRecoveryTransportRequestHandler());
}
 
Example 9
Source Project: crate   Source File: SyncedFlushService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void onShardInactive(final IndexShard indexShard) {
    // we only want to call sync flush once, so only trigger it when we are on a primary
    if (indexShard.routingEntry().primary()) {
        attemptSyncedFlush(indexShard.shardId(), new ActionListener<ShardsSyncedFlushResult>() {
            @Override
            public void onResponse(ShardsSyncedFlushResult syncedFlushResult) {
                LOGGER.trace("{} sync flush on inactive shard returned successfully for sync_id: {}", syncedFlushResult.getShardId(), syncedFlushResult.syncId());
            }

            @Override
            public void onFailure(Exception e) {
                LOGGER.debug(() -> new ParameterizedMessage("{} sync flush on inactive shard failed", indexShard.shardId()), e);
            }
        });
    }
}
 
Example 10
Source Project: crate   Source File: TransportShardDeleteActionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void prepare() throws Exception {
    indexUUID = UUIDs.randomBase64UUID();
    IndicesService indicesService = mock(IndicesService.class);
    IndexService indexService = mock(IndexService.class);
    when(indicesService.indexServiceSafe(new Index(TABLE_IDENT.indexNameOrAlias(), indexUUID))).thenReturn(indexService);
    indexShard = mock(IndexShard.class);
    when(indexService.getShard(0)).thenReturn(indexShard);


    transportShardDeleteAction = new TransportShardDeleteAction(
        MockTransportService.createNewService(
            Settings.EMPTY, Version.CURRENT, THREAD_POOL, clusterService.getClusterSettings()),
        mock(IndexNameExpressionResolver.class),
        mock(ClusterService.class),
        indicesService,
        mock(ThreadPool.class),
        mock(ShardStateAction.class),
        mock(SchemaUpdateClient.class)
    );
}
 
Example 11
Source Project: crate   Source File: BlobRecoveryHandler.java    License: Apache License 2.0 6 votes vote down vote up
public BlobRecoveryHandler(IndexShard shard,
                           RecoveryTargetHandler recoveryTarget,
                           StartRecoveryRequest request,
                           int fileChunkSizeInBytes,
                           int maxConcurrentFileChunks,
                           final TransportService transportService,
                           BlobTransferTarget blobTransferTarget,
                           BlobIndicesService blobIndicesService) {
    super(shard, recoveryTarget, request, fileChunkSizeInBytes, maxConcurrentFileChunks);
    assert BlobIndex.isBlobIndex(shard.shardId().getIndexName()) : "Shard must belong to a blob index";
    this.blobShard = blobIndicesService.blobShardSafe(request.shardId());
    this.request = request;
    this.transportService = transportService;
    this.blobTransferTarget = blobTransferTarget;
    this.shard = shard;
    String property = System.getProperty("tests.short_timeouts");
    if (property == null) {
        GET_HEAD_TIMEOUT = 30;
    } else {
        GET_HEAD_TIMEOUT = 2;
    }
}
 
Example 12
Source Project: crate   Source File: PeerRecoverySourceService.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Adds recovery source handler.
 */
synchronized RecoverySourceHandler addNewRecovery(StartRecoveryRequest request, IndexShard shard) {
    for (RecoverySourceHandler existingHandler : recoveryHandlers) {
        if (existingHandler.getRequest().targetAllocationId().equals(request.targetAllocationId())) {
            throw new DelayRecoveryException("recovery with same target already registered, waiting for " +
                "previous recovery attempt to be cancelled or completed");
        }
    }
    RecoverySourceHandler handler = createRecoverySourceHandler(request, shard);
    recoveryHandlers.add(handler);
    return handler;
}
 
Example 13
Source Project: ES-Fastloader   Source File: AppendLuceneTransportAction.java    License: Apache License 2.0 5 votes vote down vote up
private void doExecuteCore(AppendLuceneRequest request, ActionListener<AppendLuceneResponse> listener) {
    try {
        // 对请求做check
        request.check();

        // 获得shard信息
        ShardId shardId = new ShardId(request.indexName, request.uuid, request.shardId);
        IndexShard shard = indicesService.getShardOrNull(shardId);
        if(shard==null) {
            throw new Exception("shard not found, indexName:" + request.indexName + ", shardId:" + request.shardId);
        }

        // 获得lucene的IndexWriter对象
        /* FIXME 这里需要修改es的代码, 将lucene的IndexWriter对象暴露给plugin使用  */
        InternalEngine engine = (InternalEngine) shard.getEngineOrNull();
        IndexWriter indexWriter = engine.getIndexWriter();

        // 处理主键冲突情况
        long deleteCount = -1;
        List<String> appendDirs = request.getAppendDirs();
        if(request.primeKey!=null && request.primeKey.length()>0) {
            deleteCount = doPrimerKey(appendDirs, indexWriter, request.primeKey);
        }

        // 将新的lucene文件加入到shard中
        Directory[] indexes = new Directory[appendDirs.size()];
        for(int i=0; i<appendDirs.size(); i++) {
            indexes[i] = FSDirectory.open(Paths.get(appendDirs.get(i)));
        }
        indexWriter.addIndexes(indexes);
        indexWriter.commit();

        // 构建response
        AppendLuceneResponse response = new AppendLuceneResponse();
        response.deleteCount = deleteCount;
        listener.onResponse(response);
    } catch (Exception e) {
        listener.onFailure(e);
    }
}
 
Example 14
Source Project: crate   Source File: TransportWriteAction.java    License: Apache License 2.0 5 votes vote down vote up
public WriteReplicaResult(ReplicaRequest request, @Nullable Location location,
                          @Nullable Exception operationFailure, IndexShard replica, Logger logger) {
    super(operationFailure);
    this.location = location;
    if (operationFailure != null) {
        this.finishedAsyncActions = true;
    } else {
        new AsyncAfterWriteAction(replica, location, this).run();
    }
}
 
Example 15
Source Project: Elasticsearch   Source File: RecoverySource.java    License: Apache License 2.0 5 votes vote down vote up
synchronized void add(IndexShard shard, RecoverySourceHandler handler) {
    Set<RecoverySourceHandler> shardRecoveryHandlers = ongoingRecoveries.get(shard);
    if (shardRecoveryHandlers == null) {
        shardRecoveryHandlers = new HashSet<>();
        ongoingRecoveries.put(shard, shardRecoveryHandlers);
    }
    assert shardRecoveryHandlers.contains(handler) == false : "Handler was already registered [" + handler + "]";
    shardRecoveryHandlers.add(handler);
    shard.recoveryStats().incCurrentAsSource();
}
 
Example 16
Source Project: Elasticsearch   Source File: TransportDeleteAction.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected Tuple<DeleteResponse, DeleteRequest> shardOperationOnPrimary(MetaData metaData, DeleteRequest request) {
    IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).shardSafe(request.shardId().id());
    final WriteResult<DeleteResponse> result = executeDeleteRequestOnPrimary(request, indexShard);
    processAfterWrite(request.refresh(), indexShard, result.location);
    return new Tuple<>(result.response, request);
}
 
Example 17
Source Project: Elasticsearch   Source File: SearchContextFactory.java    License: Apache License 2.0 5 votes vote down vote up
public CrateSearchContext createContext(
        int jobSearchContextId,
        IndexShard indexshard,
        Engine.Searcher engineSearcher,
        WhereClause whereClause) {

    ShardId shardId = indexshard.shardId();
    SearchShardTarget searchShardTarget = new SearchShardTarget(
            clusterService.state().nodes().localNodeId(),
            shardId.getIndex(),
            shardId.id()
    );
    IndexService indexService = indexshard.indexService();
    CrateSearchContext searchContext = new CrateSearchContext(
            jobSearchContextId,
            System.currentTimeMillis(),
            searchShardTarget,
            engineSearcher,
            indexService,
            indexshard,
            scriptService,
            pageCacheRecycler,
            bigArrays,
            threadPool.estimatedTimeInMillisCounter(),
            Optional.<Scroll>absent()
    );
    LuceneQueryBuilder.Context context = luceneQueryBuilder.convert(
            whereClause,  indexService.mapperService(), indexService.fieldData(), indexService.cache());
    searchContext.parsedQuery(new ParsedQuery(context.query(), EMPTY_NAMED_FILTERS));

    Float minScore = context.minScore();
    if (minScore != null) {
        searchContext.minimumScore(minScore);
    }

    return searchContext;
}
 
Example 18
Source Project: Elasticsearch   Source File: ShardCollectService.java    License: Apache License 2.0 5 votes vote down vote up
private CrateCollector getLuceneIndexCollector(ThreadPool threadPool,
                                               final RoutedCollectPhase collectPhase,
                                               final ShardProjectorChain projectorChain,
                                               final JobCollectContext jobCollectContext) throws Exception {
    SharedShardContext sharedShardContext = jobCollectContext.sharedShardContexts().getOrCreateContext(shardId);
    Engine.Searcher searcher = sharedShardContext.searcher();
    IndexShard indexShard = sharedShardContext.indexShard();
    CrateSearchContext searchContext = null;
    try {
         searchContext = searchContextFactory.createContext(
                sharedShardContext.readerId(),
                indexShard,
                searcher,
                collectPhase.whereClause()
        );
        jobCollectContext.addSearchContext(sharedShardContext.readerId(), searchContext);
        CollectInputSymbolVisitor.Context docCtx = docInputSymbolVisitor.extractImplementations(collectPhase);
        Executor executor = threadPool.executor(ThreadPool.Names.SEARCH);

        return new CrateDocCollector(
                searchContext,
                executor,
                Symbols.containsColumn(collectPhase.toCollect(), DocSysColumns.SCORE),
                jobCollectContext.queryPhaseRamAccountingContext(),
                projectorChain.newShardDownstreamProjector(projectorVisitor),
                docCtx.topLevelInputs(),
                docCtx.docLevelExpressions()
        );
    } catch (Throwable t) {
        if (searchContext == null) {
            searcher.close();
        } else {
            searchContext.close(); // will close searcher too
        }
        throw t;
    }
}
 
Example 19
Source Project: Elasticsearch   Source File: TransportIndexAction.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Utility method to create either an index or a create operation depending
 * on the {@link IndexRequest.OpType} of the request.
 */
public static Engine.IndexingOperation prepareIndexOperationOnPrimary(BulkShardRequest shardRequest, IndexRequest request, IndexShard indexShard) {
    SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, request.source()).index(request.index()).type(request.type()).id(request.id())
        .routing(request.routing()).parent(request.parent()).timestamp(request.timestamp()).ttl(request.ttl());
    boolean canHaveDuplicates = request.canHaveDuplicates();
    if (shardRequest != null) {
        canHaveDuplicates |= shardRequest.canHaveDuplicates();
    }
    if (request.opType() == IndexRequest.OpType.INDEX) {
        return indexShard.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType(), canHaveDuplicates);
    } else {
        assert request.opType() == IndexRequest.OpType.CREATE : request.opType();
        return indexShard.prepareCreateOnPrimary(sourceToParse, request.version(), request.versionType(), canHaveDuplicates, request.autoGeneratedId());
    }
}
 
Example 20
public GraphiteReporter(String host, int port, String prefix, NodeIndicesStats nodeIndicesStats,
                        List<IndexShard> indexShards, NodeStats nodeStats,
                        Pattern graphiteInclusionRegex, Pattern graphiteExclusionRegex) {
    this.host = host;
    this.port = port;
    this.prefix = prefix;
    this.indexShards = indexShards;
    this.nodeStats = nodeStats;
    this.graphiteInclusionRegex = graphiteInclusionRegex;
    this.graphiteExclusionRegex = graphiteExclusionRegex;
    this.timestamp = Long.toString(System.currentTimeMillis() / 1000);
    this.nodeIndicesStats = nodeIndicesStats;
}
 
Example 21
Source Project: crate   Source File: BlobIndex.java    License: Apache License 2.0 5 votes vote down vote up
void initializeShard(IndexShard indexShard) {
    BlobShard blobShard = shards.get(indexShard.shardId().id());
    if (blobShard == null) {
        throw new IllegalStateException("Shard needs to be created before it is initialized");
    }
    blobShard.initialize();
}
 
Example 22
Source Project: crate   Source File: IndexService.java    License: Apache License 2.0 5 votes vote down vote up
private long getAvgShardSizeInBytes() throws IOException {
    long sum = 0;
    int count = 0;
    for (IndexShard indexShard : this) {
        sum += indexShard.store().stats().sizeInBytes();
        count++;
    }
    if (count == 0) {
        return -1L;
    } else {
        return sum / count;
    }
}
 
Example 23
Source Project: crate   Source File: RecoverySourceHandlerTests.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testVerifySeqNoStatsWhenRecoverWithSyncId() throws Exception {
    IndexShard shard = mock(IndexShard.class);
    when(shard.state()).thenReturn(IndexShardState.STARTED);
    RecoverySourceHandler handler = new RecoverySourceHandler(
        shard, new TestRecoveryTargetHandler(), getStartRecoveryRequest(), between(1, 16), between(1, 4));

    String syncId = UUIDs.randomBase64UUID();
    int numDocs = between(0, 1000);
    long localCheckpoint = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE);
    long maxSeqNo = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE);
    assertTrue(handler.canSkipPhase1(
        newMetadataSnapshot(syncId, Long.toString(localCheckpoint), Long.toString(maxSeqNo), numDocs),
        newMetadataSnapshot(syncId, Long.toString(localCheckpoint), Long.toString(maxSeqNo), numDocs)));

    Class<? extends Throwable> expectedError = Assertions.ENABLED ? AssertionError.class : IllegalStateException.class;
    Throwable error = expectThrows(expectedError, () -> {
        long localCheckpointOnTarget = randomValueOtherThan(
            localCheckpoint,
            () -> randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE));
        long maxSeqNoOnTarget = randomValueOtherThan(
            maxSeqNo,
            () -> randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE));
        handler.canSkipPhase1(
            newMetadataSnapshot(syncId, Long.toString(localCheckpoint), Long.toString(maxSeqNo), numDocs),
            newMetadataSnapshot(syncId, Long.toString(localCheckpointOnTarget), Long.toString(maxSeqNoOnTarget), numDocs));
    });
    assertThat(error.getMessage(), containsString("try to recover [index][1] with sync id but seq_no stats are mismatched:"));
}
 
Example 24
Source Project: Elasticsearch   Source File: CrateSearchContext.java    License: Apache License 2.0 5 votes vote down vote up
private CrateSearchShardRequest(long nowInMillis, Optional<Scroll> scroll,
                                IndexShard indexShard) {
    this.nowInMillis = nowInMillis;
    this.scroll = scroll.orNull();
    this.index = indexShard.indexService().index().name();
    this.shardId = indexShard.shardId().id();
}
 
Example 25
Source Project: crate   Source File: RecoverySourceHandlerTests.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSendSnapshotStopOnError() throws Exception {
    final int fileChunkSizeInBytes = between(1, 10 * 1024);
    final StartRecoveryRequest request = getStartRecoveryRequest();
    final IndexShard shard = mock(IndexShard.class);
    when(shard.state()).thenReturn(IndexShardState.STARTED);
    final List<Translog.Operation> ops = new ArrayList<>();
    for (int numOps = between(1, 256), i = 0; i < numOps; i++) {
        final Engine.Index index = getIndex(Integer.toString(i));
        ops.add(new Translog.Index(index, new Engine.IndexResult(1, 1, i, true)));
    }
    final AtomicBoolean wasFailed = new AtomicBoolean();
    RecoveryTargetHandler recoveryTarget = new TestRecoveryTargetHandler() {
        @Override
        public void indexTranslogOperations(List<Translog.Operation> operations, int totalTranslogOps, long timestamp,
                                            long msu, ActionListener<Long> listener) {
            if (randomBoolean()) {
                maybeExecuteAsync(() -> listener.onResponse(SequenceNumbers.NO_OPS_PERFORMED));
            } else {
                maybeExecuteAsync(() -> listener.onFailure(new RuntimeException("test - failed to index")));
                wasFailed.set(true);
            }
        }
    };
    RecoverySourceHandler handler = new RecoverySourceHandler(shard, recoveryTarget, request, fileChunkSizeInBytes, between(1, 10));
    PlainActionFuture<RecoverySourceHandler.SendSnapshotResult> future = new PlainActionFuture<>();
    final long startingSeqNo = randomLongBetween(0, ops.size() - 1L);
    final long endingSeqNo = randomLongBetween(startingSeqNo, ops.size() - 1L);
    handler.phase2(startingSeqNo, startingSeqNo, endingSeqNo, newTranslogSnapshot(ops, Collections.emptyList()),
                   randomNonNegativeLong(), randomNonNegativeLong(), future);
    if (wasFailed.get()) {
        assertThat(expectThrows(RuntimeException.class, future::actionGet).getMessage(), equalTo("test - failed to index"));
    }
}
 
Example 26
Source Project: crate   Source File: ShardCollectorProvider.java    License: Apache License 2.0 5 votes vote down vote up
ShardCollectorProvider(ClusterService clusterService,
                       Schemas schemas,
                       NodeJobsCounter nodeJobsCounter,
                       Functions functions,
                       ThreadPool threadPool,
                       Settings settings,
                       TransportActionProvider transportActionProvider,
                       IndexShard indexShard,
                       ShardRowContext shardRowContext) {
    this.shardRowContext = shardRowContext;
    shardNormalizer = new EvaluatingNormalizer(
        functions,
        RowGranularity.SHARD,
        new ShardReferenceResolver(schemas, shardRowContext),
        null
    );
    projectorFactory = new ProjectionToProjectorVisitor(
        clusterService,
        nodeJobsCounter,
        functions,
        threadPool,
        settings,
        transportActionProvider,
        new InputFactory(functions),
        shardNormalizer,
        t -> null,
        t -> null,
        indexShard.indexSettings().getIndexVersionCreated(),
        indexShard.shardId()
    );
}
 
Example 27
Source Project: crate   Source File: ShardSegments.java    License: Apache License 2.0 5 votes vote down vote up
private Stream<ShardSegment> buildShardSegment(IndexShard indexShard) {
    try {
        List<Segment> segments = indexShard.segments(false);
        ShardId shardId = indexShard.shardId();
        return segments.stream().map(
            sgmt -> new ShardSegment(shardId.getId(),
                                     shardId.getIndexName(),
                                     sgmt,
                                     indexShard.routingEntry().primary()));
    } catch (AlreadyClosedException ignored) {
        return Stream.empty();
    }
}
 
Example 28
Source Project: crate   Source File: PeerRecoverySourceService.java    License: Apache License 2.0 5 votes vote down vote up
private RecoverySourceHandler createRecoverySourceHandler(StartRecoveryRequest request, IndexShard shard) {
    RecoverySourceHandler handler;
    final RemoteRecoveryTargetHandler recoveryTarget = new RemoteRecoveryTargetHandler(
        request.recoveryId(),
        request.shardId(),
        transportService,
        request.targetNode(),
        recoverySettings,
        throttleTime -> shard.recoveryStats().addThrottleTime(throttleTime));

    // CRATE_PATCH: used to inject BlobRecoveryHandler
    int recoveryChunkSizeInBytes = recoverySettings.getChunkSize().bytesAsInt();
    handler = getCustomRecoverySourceHandler(
        shard,
        recoveryTarget,
        request,
        recoveryChunkSizeInBytes
    );

    if (handler != null) {
        return handler;
    } else {
        return new RecoverySourceHandler(
            shard,
            recoveryTarget,
            request,
            recoveryChunkSizeInBytes,
            recoverySettings.getMaxConcurrentFileChunks());
    }
}
 
Example 29
Source Project: crate   Source File: IndexService.java    License: Apache License 2.0 5 votes vote down vote up
private void maybeRefreshEngine() {
    if (indexSettings.getRefreshInterval().millis() > 0) {
        for (IndexShard shard : this.shards.values()) {
            if (shard.isReadAllowed()) {
                try {
                    if (shard.isRefreshNeeded()) {
                        shard.refresh("schedule");
                    }
                } catch (IndexShardClosedException | AlreadyClosedException ex) {
                    // fine - continue;
                }
            }
        }
    }
}
 
Example 30
Source Project: Elasticsearch   Source File: InternalIndicesLifecycle.java    License: Apache License 2.0 5 votes vote down vote up
public void shardRoutingChanged(IndexShard indexShard, @Nullable ShardRouting oldRouting, ShardRouting newRouting) {
    for (Listener listener : listeners) {
        try {
            listener.shardRoutingChanged(indexShard, oldRouting, newRouting);
        } catch (Throwable t) {
            logger.warn("{} failed to invoke shard touring changed callback", t, indexShard.shardId());
        }
    }
}