Java Code Examples for org.elasticsearch.index.shard.ShardId

@Override
public void logDocumentDeleted(ShardId shardId, Delete delete, DeleteResult result) {

    String effectiveUser = getUser();

    if(!checkComplianceFilter(Category.COMPLIANCE_DOC_WRITE, effectiveUser, getOrigin())) {
        return;
    }

    AuditMessage msg = new AuditMessage(Category.COMPLIANCE_DOC_WRITE, clusterService, getOrigin(), null);
    TransportAddress remoteAddress = getRemoteAddress();
    msg.addRemoteAddress(remoteAddress);
    msg.addEffectiveUser(effectiveUser);
    msg.addIndices(new String[]{shardId.getIndexName()});
    msg.addResolvedIndices(new String[]{shardId.getIndexName()});
    msg.addId(delete.id());
    msg.addShardId(shardId);
    msg.addComplianceDocVersion(result.getVersion());
    msg.addComplianceOperation(Operation.DELETE);
    save(msg);
}
 

@Test
public void testSearchWithScores() throws Exception {
    IndexWriter w = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
    KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
    fieldType.setName("x");
    fieldType.freeze();

    for (int i = 0; i < 3; i++) {
        addDoc(w, fieldType, "Arthur");
    }
    addDoc(w, fieldType, "Arthur"); // not "Arthur" to lower score
    w.commit();
    IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(w, true, true));

    List<LuceneCollectorExpression<?>> columnReferences = Collections.singletonList(new ScoreCollectorExpression());
    Query query = fieldType.termsQuery(Collections.singletonList("Arthur"), null);
    LuceneOrderedDocCollector collector = collector(searcher, columnReferences, query, null, true);
    KeyIterable<ShardId, Row> result = collector.collect();

    assertThat(Iterables.size(result), is(2));

    Iterator<Row> values = result.iterator();

    assertThat(values.next().get(0), Matchers.is(1.0F));
    assertThat(values.next().get(0), Matchers.is(1.0F));
}
 

public ProjectionToProjectorVisitor(ClusterService clusterService,
                                    NodeJobsCounter nodeJobsCounter,
                                    Functions functions,
                                    ThreadPool threadPool,
                                    Settings settings,
                                    TransportActionProvider transportActionProvider,
                                    InputFactory inputFactory,
                                    EvaluatingNormalizer normalizer,
                                    Function<RelationName, SysRowUpdater<?>> sysUpdaterGetter,
                                    Function<RelationName, StaticTableDefinition<?>> staticTableDefinitionGetter,
                                    Version indexVersionCreated,
                                    @Nullable ShardId shardId) {
    this.clusterService = clusterService;
    this.nodeJobsCounter = nodeJobsCounter;
    this.functions = functions;
    this.threadPool = threadPool;
    this.settings = settings;
    this.transportActionProvider = transportActionProvider;
    this.inputFactory = inputFactory;
    this.normalizer = normalizer;
    this.sysUpdaterGetter = sysUpdaterGetter;
    this.staticTableDefinitionGetter = staticTableDefinitionGetter;
    this.indexVersionCreated = indexVersionCreated;
    this.shardId = shardId;
    this.numProcessors = EsExecutors.numberOfProcessors(settings);
}
 

/**
 * Returns the expected shard size for the given shard or the default value provided if not enough information are available
 * to estimate the shards size.
 */
public static long getExpectedShardSize(ShardRouting shard,
                                        long defaultValue,
                                        ClusterInfo clusterInfo,
                                        MetaData metaData,
                                        RoutingTable routingTable) {
    final IndexMetaData indexMetaData = metaData.getIndexSafe(shard.index());
    if (indexMetaData.getResizeSourceIndex() != null && shard.active() == false &&
        shard.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) {
        // in the shrink index case we sum up the source index shards since we basically make a copy of the shard in
        // the worst case
        long targetShardSize = 0;
        final Index mergeSourceIndex = indexMetaData.getResizeSourceIndex();
        final IndexMetaData sourceIndexMeta = metaData.index(mergeSourceIndex);
        if (sourceIndexMeta != null) {
            final Set<ShardId> shardIds = IndexMetaData.selectRecoverFromShards(shard.id(),
                sourceIndexMeta, indexMetaData.getNumberOfShards());
            for (IndexShardRoutingTable shardRoutingTable : routingTable.index(mergeSourceIndex.getName())) {
                if (shardIds.contains(shardRoutingTable.shardId())) {
                    targetShardSize += clusterInfo.getShardSize(shardRoutingTable.primaryShard(), 0);
                }
            }
        }
        return targetShardSize == 0 ? defaultValue : targetShardSize;
    } else {
        return clusterInfo.getShardSize(shard, defaultValue);
    }
}
 

public RecoveryState(StreamInput in) throws IOException {
    timer = new Timer(in);
    stage = Stage.fromId(in.readByte());
    shardId = new ShardId(in);
    recoverySource = RecoverySource.readFrom(in);
    targetNode = new DiscoveryNode(in);
    sourceNode = in.readOptionalWriteable(DiscoveryNode::new);
    index = new Index(in);
    translog = new Translog(in);
    verifyIndex = new VerifyIndex(in);
    primary = in.readBoolean();
}
 

public ShardId getShardId() {
    List<String> shard = getMetadata(SHARD_METADATA_KEY);
    if (shard != null && shard.isEmpty() == false) {
        return new ShardId(getIndex(), Integer.parseInt(shard.get(0)));
    }
    return null;
}
 

public RecoverFilesRecoveryException(ShardId shardId, int numberOfFiles, ByteSizeValue totalFilesSize, Throwable cause) {
    super("Failed to transfer [{}] files with total size of [{}]", cause, numberOfFiles, totalFilesSize);
    Objects.requireNonNull(totalFilesSize, "totalFilesSize must not be null");
    setShard(shardId);
    this.numberOfFiles = numberOfFiles;
    this.totalFilesSize = totalFilesSize;
}
 

@Override
public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard,
                                   Settings indexSettings) {
    if (indexShard != null) {
        ongoingRecoveries.cancel(indexShard, "shard is closed");
    }
}
 

/**
 * Initializes a new empty index, with an option to control if its from an API or not.
 */
private Builder initializeEmpty(IndexMetaData indexMetaData, UnassignedInfo unassignedInfo) {
    if (!shards.isEmpty()) {
        throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created");
    }
    for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) {
        IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.getIndex(), shardId));
        for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) {
            indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, i == 0, unassignedInfo));
        }
        shards.put(shardId, indexShardRoutingBuilder.build());
    }
    return this;
}
 

public DlsFlsDirectoryReader(final DirectoryReader in, final Set<String> includes, final Query dlsQuery,
                             final IndexService indexService, final ThreadContext threadContext,
                             final ClusterService clusterService, final ComplianceConfig complianceConfig,
                             final AuditLog auditlog, final Set<String> maskedFields, ShardId shardId) throws IOException {
    super(in, new DlsFlsSubReaderWrapper(includes, dlsQuery, indexService, threadContext, clusterService, complianceConfig, auditlog, maskedFields, shardId));
    this.includes = includes;
    this.dlsQuery = dlsQuery;
    this.indexService = indexService;
    this.threadContext = threadContext;
    this.clusterService = clusterService;
    this.complianceConfig = complianceConfig;
    this.auditlog = auditlog;
    this.maskedFields = maskedFields;
    this.shardId = shardId;
}
 

public static IndexShardRoutingTable readFromThin(StreamInput in, Index index) throws IOException {
    int iShardId = in.readVInt();
    ShardId shardId = new ShardId(index, iShardId);
    Builder builder = new Builder(shardId);

    int size = in.readVInt();
    for (int i = 0; i < size; i++) {
        ShardRouting shard = new ShardRouting(shardId, in);
        builder.addShard(shard);
    }

    return builder.build();
}
 

@Override
public void afterIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) {
    if (indexShard != null) {
        Boolean remove = shardSet.remove(indexShard);
        if (remove == Boolean.TRUE) {
            Logger logger = Loggers.getLogger(getClass(), indexShard.shardId());
            MockFSDirectoryService.checkIndex(logger, indexShard.store(), indexShard.shardId());
        }
    }
}
 

/**
 * failure constructor
 */
public ShardsSyncedFlushResult(ShardId shardId, int totalShards, String failureReason) {
    this.syncId = null;
    this.failureReason = failureReason;
    this.shardResponses = ImmutableMap.of();
    this.shardId = shardId;
    this.totalShards = totalShards;
}
 

@Override
public synchronized void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard,
                                                Settings indexSettings) {
    if (indexShard != null) {
        getStats.addTotals(indexShard.getStats());
        indexingStats.addTotals(indexShard.indexingStats());
        searchStats.addTotals(indexShard.searchStats());
        mergeStats.addTotals(indexShard.mergeStats());
        refreshStats.addTotals(indexShard.refreshStats());
        flushStats.addTotals(indexShard.flushStats());
        recoveryStats.addTotals(indexShard.recoveryStats());
    }
}
 

/**
 * Fails the given snapshot restore operation for the given shard
 */
public void failRestore(SnapshotId snapshotId, ShardId shardId) {
    logger.debug("[{}] failed to restore shard  [{}]", snapshotId, shardId);
    UpdateIndexShardRestoreStatusRequest request = new UpdateIndexShardRestoreStatusRequest(snapshotId, shardId,
            new ShardRestoreStatus(clusterService.state().nodes().localNodeId(), RestoreInProgress.State.FAILURE));
        transportService.sendRequest(clusterService.state().nodes().masterNode(),
                UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
}
 

@Override
protected void resolveRequest(MetaData metaData, String concreteIndex, IndexRequest request) {
    MappingMetaData mappingMd = null;
    if (metaData.hasIndex(concreteIndex)) {
        mappingMd = metaData.index(concreteIndex).mappingOrDefault(request.type());
    }
    request.process(metaData, mappingMd, allowIdGeneration, concreteIndex);
    ShardId shardId = clusterService.operationRouting().shardId(clusterService.state(), concreteIndex, request.type(), request.id(), request.routing());
    request.setShardId(shardId);
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    if (in.readBoolean()) {
        internalShardId = ShardId.readShardId(in);
    }
    index = in.readOptionalString();
    // no need to pass threading over the network, they are always false when coming throw a thread pool
}
 

/**
 * Returns shard for the specified id if it exists otherwise returns <code>null</code>.
 */
default T getShardOrNull(ShardId shardId) {
    U indexRef = indexService(shardId.getIndex());
    if (indexRef != null) {
        return indexRef.getShardOrNull(shardId.id());
    }
    return null;
}
 

/**
 * Creates a new {@link org.elasticsearch.index.engine.EngineConfig}
 */
public EngineConfig(ShardId shardId, ThreadPool threadPool, ShardIndexingService indexingService,
                    Settings indexSettings, IndicesWarmer warmer, Store store, SnapshotDeletionPolicy deletionPolicy,
                    MergePolicy mergePolicy, MergeSchedulerConfig mergeSchedulerConfig, Analyzer analyzer,
                    Similarity similarity, CodecService codecService, Engine.FailedEngineListener failedEngineListener,
                    TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, IndexSearcherWrappingService wrappingService, TranslogConfig translogConfig) {
    this.shardId = shardId;
    this.indexSettings = indexSettings;
    this.threadPool = threadPool;
    this.indexingService = indexingService;
    this.warmer = warmer;
    this.store = store;
    this.deletionPolicy = deletionPolicy;
    this.mergePolicy = mergePolicy;
    this.mergeSchedulerConfig = mergeSchedulerConfig;
    this.analyzer = analyzer;
    this.similarity = similarity;
    this.codecService = codecService;
    this.failedEngineListener = failedEngineListener;
    this.wrappingService = wrappingService;
    this.optimizeAutoGenerateId = indexSettings.getAsBoolean(EngineConfig.INDEX_OPTIMIZE_AUTOGENERATED_ID_SETTING, false);
    this.compoundOnFlush = indexSettings.getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, compoundOnFlush);
    codecName = indexSettings.get(EngineConfig.INDEX_CODEC_SETTING, EngineConfig.DEFAULT_CODEC_NAME);
    // We start up inactive and rely on IndexingMemoryController to give us our fair share once we start indexing:
    indexingBufferSize = IndexingMemoryController.INACTIVE_SHARD_INDEXING_BUFFER;
    gcDeletesInMillis = indexSettings.getAsTime(INDEX_GC_DELETES_SETTING, EngineConfig.DEFAULT_GC_DELETES).millis();
    versionMapSizeSetting = indexSettings.get(INDEX_VERSION_MAP_SIZE, DEFAULT_VERSION_MAP_SIZE);
    updateVersionMapSize();
    this.translogRecoveryPerformer = translogRecoveryPerformer;
    this.forceNewTranslog = indexSettings.getAsBoolean(INDEX_FORCE_NEW_TRANSLOG, false);
    this.queryCache = queryCache;
    this.queryCachingPolicy = queryCachingPolicy;
    this.translogConfig = translogConfig;
}
 

/**
 * success constructor
 */
public ShardsSyncedFlushResult(ShardId shardId, String syncId, int totalShards, Map<ShardRouting, SyncedFlushService.ShardSyncedFlushResponse> shardResponses) {
    this.failureReason = null;
    ImmutableMap.Builder<ShardRouting, SyncedFlushService.ShardSyncedFlushResponse> builder = ImmutableMap.builder();
    this.shardResponses = builder.putAll(shardResponses).build();
    this.syncId = syncId;
    this.totalShards = totalShards;
    this.shardId = shardId;
}
 

public DeleteProjector(ClusterService clusterService,
                       IndexNameExpressionResolver indexNameExpressionResolver,
                       Settings settings,
                       ShardId shardId,
                       TransportActionProvider transportActionProvider,
                       BulkRetryCoordinatorPool bulkRetryCoordinatorPool,
                       CollectExpression<Row, ?> collectUidExpression,
                       UUID jobId) {
    super(clusterService, settings, shardId, transportActionProvider, bulkRetryCoordinatorPool,
            collectUidExpression, jobId);
    this.indexNameExpressionResolver = indexNameExpressionResolver;
}
 

@Override
public void afterIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard,
                                  Settings indexSettings) {
    for (IndexEventListener listener : listeners) {
        try {
            listener.afterIndexShardClosed(shardId, indexShard, indexSettings);
        } catch (Exception e) {
            logger.warn(() -> new ParameterizedMessage("[{}] failed to invoke after shard closed callback", shardId.getId()), e);
            throw e;
        }
    }
}
 

/**
 * On the first call this will do an initial search and provide {@link #batchSize} number of rows
 * (or less if there aren't more available)
 * </p>
 * On subsequent calls it will return more rows (max {@link #batchSize} or less.
 * These rows are always the rows that come after the last row of the previously returned rows
 * <p/>
 * Basically, calling this function multiple times pages through the shard in batches.
 */
@Override
public KeyIterable<ShardId, Row> collect() {
    try {
        if (lastDoc == null) {
            return initialSearch();
        }
        return searchMore();
    } catch (Exception e) {
        Exceptions.rethrowUnchecked(e);
        return null;
    }
}
 

/**
 * Checks if any shards were processed that the new master doesn't know about
 */
private void syncShardStatsOnNewMaster(ClusterChangedEvent event) {
    SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE);
    if (snapshotsInProgress == null) {
        return;
    }

    for (SnapshotsInProgress.Entry snapshot : snapshotsInProgress.entries()) {
        if (snapshot.state() == State.STARTED || snapshot.state() == State.ABORTED) {
            Map<ShardId, IndexShardSnapshotStatus> localShards = currentSnapshotShards(snapshot.snapshot());
            if (localShards != null) {
                ImmutableOpenMap<ShardId, ShardSnapshotStatus> masterShards = snapshot.shards();
                for (Map.Entry<ShardId, IndexShardSnapshotStatus> localShard : localShards.entrySet()) {
                    ShardId shardId = localShard.getKey();
                    ShardSnapshotStatus masterShard = masterShards.get(shardId);
                    if (masterShard != null && masterShard.state().completed() == false) {
                        final IndexShardSnapshotStatus.Copy indexShardSnapshotStatus = localShard.getValue().asCopy();
                        final Stage stage = indexShardSnapshotStatus.getStage();
                        // Master knows about the shard and thinks it has not completed
                        if (stage == Stage.DONE) {
                            // but we think the shard is done - we need to make new master know that the shard is done
                            LOGGER.debug("[{}] new master thinks the shard [{}] is not completed but the shard is done locally, " +
                                "updating status on the master", snapshot.snapshot(), shardId);
                            notifySuccessfulSnapshotShard(snapshot.snapshot(), shardId, localShard.getValue().generation());

                        } else if (stage == Stage.FAILURE) {
                            // but we think the shard failed - we need to make new master know that the shard failed
                            LOGGER.debug("[{}] new master thinks the shard [{}] is not completed but the shard failed locally, " +
                                "updating status on master", snapshot.snapshot(), shardId);
                            notifyFailedSnapshotShard(snapshot.snapshot(), shardId, indexShardSnapshotStatus.getFailure());
                        }
                    }
                }
            }
        }
    }
}
 

PKLookupTask(UUID jobId,
             int phaseId,
             String name,
             RamAccounting ramAccounting,
             Function<RamAccounting, MemoryManager> memoryManagerFactory,
             int ramAccountingBlockSizeInBytes,
             TransactionContext txnCtx,
             InputFactory inputFactory,
             PKLookupOperation pkLookupOperation,
             List<ColumnIdent> partitionedByColumns,
             List<Symbol> toCollect,
             Map<ShardId, List<PKAndVersion>> idsByShard,
             Collection<? extends Projection> shardProjections,
             RowConsumer consumer) {
    super(phaseId);
    this.jobId = jobId;
    this.name = name;
    this.ramAccounting = ramAccounting;
    this.txnCtx = txnCtx;
    this.pkLookupOperation = pkLookupOperation;
    this.idsByShard = idsByShard;
    this.shardProjections = shardProjections;
    this.consumer = consumer;
    this.memoryManagerFactory = memoryManagerFactory;
    this.ramAccountingBlockSizeInBytes = ramAccountingBlockSizeInBytes;

    this.ignoreMissing = !partitionedByColumns.isEmpty();
    DocRefResolver docRefResolver = new DocRefResolver(partitionedByColumns);

    InputFactory.Context<CollectExpression<Doc, ?>> ctx = inputFactory.ctxForRefs(txnCtx, docRefResolver);
    ctx.add(toCollect);
    expressions = ctx.expressions();
    inputRow = new InputRow(ctx.topLevelInputs());
}
 

@Nullable
public BlobShard blobShard(ShardId shardId) {
    BlobIndex blobIndex = indices.get(shardId.getIndexName());
    if (blobIndex == null) {
        return null;
    }
    return blobIndex.getShard(shardId.id());
}
 

/**
 * Return the shard location respecting global blobs data path value
 */
public File shardLocation(ShardId shardId) {
    if (blobsPath == null) {
        return new File(nodeEnvironment.availableShardPaths(shardId)[0].toFile(), BLOBS_SUB_PATH);
    }
    return shardLocation(shardId, blobsPath);
}
 

@Override
protected void shardOperationOnReplica(IndexRequest request) {
    final ShardId shardId = request.shardId();
    IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
    IndexShard indexShard = indexService.shardSafe(shardId.id());
    indexShard.checkDiskSpace(fsService);
    final Engine.IndexingOperation operation = executeIndexRequestOnReplica(request, indexShard);
    processAfterWrite(request.refresh(), indexShard, operation.getTranslogLocation());
}
 

@Inject
public IndexVersionShardService(ShardId shardId, Settings indexSettings, IndexShard indexShard) {
  super(shardId, indexSettings);
  this.indexShard = indexShard;
  this.versioningIndexingOperationListener = new VersioningIndexingOperationListener();
  indexShard.indexingService().addListener(versioningIndexingOperationListener);
  version = new AtomicLong(System.nanoTime()); // initialise version number based on time to ensure uniqueness even if shard restarted
}
 

/**
 * Increases the primary term if {@link #increasePrimaryTerm} was called for this shard id.
 */
private IndexMetaData.Builder updatePrimaryTerm(IndexMetaData oldIndexMetaData, IndexMetaData.Builder indexMetaDataBuilder,
                                                ShardId shardId, Updates updates) {
    if (updates.increaseTerm) {
        if (indexMetaDataBuilder == null) {
            indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData);
        }
        indexMetaDataBuilder.primaryTerm(shardId.id(), oldIndexMetaData.primaryTerm(shardId.id()) + 1);
    }
    return indexMetaDataBuilder;
}