org.elasticsearch.action.bulk.BackoffPolicy Java Examples

The following examples show how to use org.elasticsearch.action.bulk.BackoffPolicy. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: Elasticsearch5ApiCallBridge.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}

Example #2

Source File: BulkProcessingOptionsBuilderTest.java From ElasticUtils with MIT License

6 votes

@Test
public void custom_values_set_when_custom_values_set_for_builder() {

    // Build:
    BulkProcessingOptions options = new BulkProcessingOptionsBuilder()
            .setName("Test")
            .setConcurrentRequests(3)
            .setBulkActions(10)
            .setBulkSize(new ByteSizeValue(2, ByteSizeUnit.MB))
            .setFlushInterval(new TimeValue(321))
            .setBackoffPolicy(BackoffPolicy.noBackoff())
            .build();

    // Check Values:
    Assert.assertEquals("Test", options.getName());
    Assert.assertEquals(3, options.getConcurrentRequests());
    Assert.assertEquals(10, options.getBulkActions());
    Assert.assertEquals(new ByteSizeValue(2, ByteSizeUnit.MB).bytes(), options.getBulkSize().bytes());
    Assert.assertEquals(new TimeValue(321).getMillis(), options.getFlushInterval().getMillis());
    Assert.assertEquals(BackoffPolicy.noBackoff().getClass(), options.getBackoffPolicy().getClass());
}

Example #3

Source File: ElasticSearchClient.java From skywalking with Apache License 2.0

6 votes

public BulkProcessor createBulkProcessor(int bulkActions, int flushInterval, int concurrentRequests) {
    BulkProcessor.Listener listener = createBulkListener();

    return BulkProcessor.builder(client::bulkAsync, listener)
                        .setBulkActions(bulkActions)
                        .setFlushInterval(TimeValue.timeValueSeconds(flushInterval))
                        .setConcurrentRequests(concurrentRequests)
                        .setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 3))
                        .build();
}

Example #4

Source File: Elasticsearch7ApiCallBridge.java From flink with Apache License 2.0

6 votes

@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}

Example #5

Source File: Elasticsearch5ApiCallBridge.java From flink with Apache License 2.0

6 votes

@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}

Example #6

Source File: Elasticsearch6ApiCallBridge.java From flink with Apache License 2.0

6 votes

@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}

Example #7

Source File: TransportDistributedResultAction.java From crate with Apache License 2.0

6 votes

@Inject
public TransportDistributedResultAction(Transports transports,
                                        TasksService tasksService,
                                        ThreadPool threadPool,
                                        TransportService transportService,
                                        ClusterService clusterService,
                                        TransportKillJobsNodeAction killJobsAction,
                                        Settings settings) {
    this(transports,
        tasksService,
        threadPool,
        transportService,
        clusterService,
        killJobsAction,
        BackoffPolicy.exponentialBackoff());
}

Example #8

Source File: TransportDistributedResultAction.java From crate with Apache License 2.0

6 votes

@VisibleForTesting
TransportDistributedResultAction(Transports transports,
                                 TasksService tasksService,
                                 ThreadPool threadPool,
                                 TransportService transportService,
                                 ClusterService clusterService,
                                 TransportKillJobsNodeAction killJobsAction,
                                 BackoffPolicy backoffPolicy) {
    this.transports = transports;
    this.tasksService = tasksService;
    scheduler = threadPool.scheduler();
    this.clusterService = clusterService;
    this.killJobsAction = killJobsAction;
    this.backoffPolicy = backoffPolicy;

    transportService.registerRequestHandler(DISTRIBUTED_RESULT_ACTION,
        DistributedResultRequest::new,
        ThreadPool.Names.SAME, // <- we will dispatch later at the nodeOperation on non failures
        new NodeActionRequestHandler<>(this));
}

Example #9

Source File: BatchIteratorBackpressureExecutor.java From crate with Apache License 2.0

6 votes

/**
 * @param batchIterator provides the items for {@code execute}
 * @param execute async function which is called for each item from the batchIterator
 * @param combiner used to combine partial results returned by the {@code execute} function
 * @param identity default value (this is the result, if the batchIterator contains no items)
 * @param pauseConsumption predicate used to check if the consumption should be paused
 */
public BatchIteratorBackpressureExecutor(ScheduledExecutorService scheduler,
                                         Executor executor,
                                         BatchIterator<T> batchIterator,
                                         Function<T, CompletableFuture<R>> execute,
                                         BinaryOperator<R> combiner,
                                         R identity,
                                         Predicate<T> pauseConsumption,
                                         BackoffPolicy backoffPolicy) {
    this.executor = executor;
    this.batchIterator = batchIterator;
    this.scheduler = scheduler;
    this.execute = execute;
    this.combiner = combiner;
    this.pauseConsumption = pauseConsumption;
    this.throttleDelay = backoffPolicy.iterator();
    this.resultRef = new AtomicReference<>(identity);
    this.continueConsumptionOrFinish = this::continueConsumptionOrFinish;
}

Example #10

Source File: BulkProcessorBulider.java From flume-elasticsearch-sink with Apache License 2.0

6 votes

private BulkProcessor build(final RestHighLevelClient client) {
    logger.trace("Bulk processor name: [{}]  bulkActions: [{}], bulkSize: [{}], flush interval time: [{}]," +
                    " concurrent Request: [{}], backoffPolicyTimeInterval: [{}], backoffPolicyRetries: [{}] ",
            new Object[]{bulkProcessorName, bulkActions, bulkSize, flushIntervalTime,
                    concurrentRequest, backoffPolicyTimeInterval, backoffPolicyRetries});
    BiConsumer<BulkRequest, ActionListener<BulkResponse>> bulkConsumer =
            (request, bulkListener) -> client
                    .bulkAsync(request, RequestOptions.DEFAULT, bulkListener);
    return BulkProcessor.builder(bulkConsumer, getListener())
            .setBulkActions(bulkActions)
            .setBulkSize(bulkSize)
            .setFlushInterval(flushIntervalTime)
            .setConcurrentRequests(concurrentRequest)
            .setBackoffPolicy(BackoffPolicy.exponentialBackoff(
                    Util.getTimeValue(backoffPolicyTimeInterval,
                            DEFAULT_ES_BACKOFF_POLICY_START_DELAY),
                    backoffPolicyRetries))
            .build();
}

Example #11

Source File: Elasticsearch2ApiCallBridge.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}

Example #12

Source File: AnomalyResultHandler.java From anomaly-detection with Apache License 2.0

6 votes

public AnomalyResultHandler(
    Client client,
    Settings settings,
    ClusterService clusterService,
    IndexNameExpressionResolver indexNameExpressionResolver,
    AnomalyDetectionIndices anomalyDetectionIndices,
    ThreadPool threadPool
) {
    this.client = client;
    this.clusterService = clusterService;
    this.indexNameExpressionResolver = indexNameExpressionResolver;
    this.anomalyDetectionIndices = anomalyDetectionIndices;
    this.threadPool = threadPool;
    this.resultSavingBackoffPolicy = BackoffPolicy
        .exponentialBackoff(
            AnomalyDetectorSettings.BACKOFF_INITIAL_DELAY.get(settings),
            AnomalyDetectorSettings.MAX_RETRY_FOR_BACKOFF.get(settings)
        );
}

Example #13

Source File: Elasticsearch6ApiCallBridge.java From flink with Apache License 2.0

6 votes

@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}

Example #14

Source File: Elasticsearch5ApiCallBridge.java From flink with Apache License 2.0

6 votes

@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}

Example #15

Source File: Elasticsearch2ApiCallBridge.java From flink with Apache License 2.0

6 votes

@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}

Example #16

Source File: Elasticsearch6ApiCallBridge.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}

Example #17

Source File: TransportDistributedResultActionTest.java From crate with Apache License 2.0

5 votes

@Test
public void testKillIsInvokedIfContextIsNotFound() throws Exception {
    TasksService tasksService = new TasksService(clusterService, new JobsLogs(() -> false));
    AtomicInteger numBroadcasts = new AtomicInteger(0);
    TransportKillJobsNodeAction killJobsAction = new TransportKillJobsNodeAction(
        tasksService,
        clusterService,
        mock(TransportService.class)
    ) {
        @Override
        public void broadcast(KillJobsRequest request, ActionListener<Long> listener, Collection<String> excludedNodeIds) {
            numBroadcasts.incrementAndGet();
        }
    };
    TransportDistributedResultAction transportDistributedResultAction = new TransportDistributedResultAction(
        mock(Transports.class),
        tasksService,
        THREAD_POOL,
        mock(TransportService.class),
        clusterService,
        killJobsAction,
        BackoffPolicy.exponentialBackoff(TimeValue.ZERO, 0)
    );

    StreamBucket.Builder builder = new StreamBucket.Builder(
        new Streamer[0], RamAccounting.NO_ACCOUNTING);
    try {
        transportDistributedResultAction.nodeOperation(
            new DistributedResultRequest(UUID.randomUUID(), 0, (byte) 0, 0, builder.build(), true)
        ).get(5, TimeUnit.SECONDS);
        fail("nodeOperation call should fail with TaskMissing");
    } catch (ExecutionException e) {
        assertThat(e.getCause(), Matchers.instanceOf(TaskMissing.class));
    }

    assertThat(numBroadcasts.get(), is(1));
}

Example #18

Source File: ElasticSearch7Client.java From skywalking with Apache License 2.0

5 votes

public BulkProcessor createBulkProcessor(int bulkActions, int flushInterval, int concurrentRequests) {
    BulkProcessor.Listener listener = createBulkListener();

    return BulkProcessor.builder(
        (bulkRequest, bulkResponseActionListener) -> client.bulkAsync(bulkRequest, RequestOptions.DEFAULT,
                                                                      bulkResponseActionListener
        ), listener)
                        .setBulkActions(bulkActions)
                        .setFlushInterval(TimeValue.timeValueSeconds(flushInterval))
                        .setConcurrentRequests(concurrentRequests)
                        .setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 3))
                        .build();
}

Example #19

Source File: BulkProcessingOptionsTest.java From ElasticUtils with MIT License

5 votes

@Test
public void get_methods_return_correct_values_when_initialized() {

    BulkProcessingOptions options = new BulkProcessingOptions("Name", 1, 2, new ByteSizeValue(10), new TimeValue(10), BackoffPolicy.exponentialBackoff());

    Assert.assertEquals("Name", options.getName());
    Assert.assertEquals(1, options.getConcurrentRequests());
    Assert.assertEquals(2, options.getBulkActions());
    Assert.assertEquals(10, options.getBulkSize().bytes());
    Assert.assertEquals(10, options.getFlushInterval().getMillis());
    Assert.assertEquals(BackoffPolicy.exponentialBackoff().getClass(), options.getBackoffPolicy().getClass());
}

Example #20

Source File: LimitedBackoffPolicyTest.java From crate with Apache License 2.0

5 votes

@Test
public void testNoNext() throws Exception {
    BackoffPolicy policy = new LimitedExponentialBackoff(0, 1, Integer.MAX_VALUE);
    Iterator<TimeValue> it = policy.iterator();
    it.next();
    expectedException.expect(NoSuchElementException.class);
    expectedException.expectMessage("Reached maximum amount of backoff iterations. Only 1 iterations allowed.");
    it.next();
}

Example #21

Source File: BulkProcessingOptionsBuilderTest.java From ElasticUtils with MIT License

5 votes

@Test
public void default_values_set_when_initializing_builder() {

    // Build:
    BulkProcessingOptions options = new BulkProcessingOptionsBuilder().build();

    // Check Values:
    Assert.assertEquals(null, options.getName());
    Assert.assertEquals(1, options.getConcurrentRequests());
    Assert.assertEquals(1000, options.getBulkActions());
    Assert.assertEquals(new ByteSizeValue(5, ByteSizeUnit.MB).bytes(), options.getBulkSize().bytes());
    Assert.assertEquals(null, options.getFlushInterval());
    Assert.assertEquals(BackoffPolicy.exponentialBackoff().getClass(), options.getBackoffPolicy().getClass());
}

Example #22

Source File: BulkProcessingOptions.java From ElasticUtils with MIT License

5 votes

public BulkProcessingOptions(String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval, BackoffPolicy backoffPolicy) {
    this.name = name;
    this.concurrentRequests = concurrentRequests;
    this.bulkActions = bulkActions;
    this.bulkSize = bulkSize;
    this.flushInterval = flushInterval;
    this.backoffPolicy = backoffPolicy;
}

Example #23

Source File: BulkProcessingOptions.java From ElasticUtils with MIT License

5 votes

public BulkProcessingOptions(String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval, BackoffPolicy backoffPolicy) {
    this.name = name;
    this.concurrentRequests = concurrentRequests;
    this.bulkActions = bulkActions;
    this.bulkSize = bulkSize;
    this.flushInterval = flushInterval;
    this.backoffPolicy = backoffPolicy;
}

Example #24

Source File: BulkProcessingOptions.java From ElasticUtils with MIT License

5 votes

public BulkProcessingOptions(String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval, BackoffPolicy backoffPolicy) {
    this.name = name;
    this.concurrentRequests = concurrentRequests;
    this.bulkActions = bulkActions;
    this.bulkSize = bulkSize;
    this.flushInterval = flushInterval;
    this.backoffPolicy = backoffPolicy;
}

Example #25

Source File: BulkProcessingOptions.java From ElasticUtils with MIT License

5 votes

public BulkProcessingOptions(String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval, BackoffPolicy backoffPolicy) {
    this.name = name;
    this.concurrentRequests = concurrentRequests;
    this.bulkActions = bulkActions;
    this.bulkSize = bulkSize;
    this.flushInterval = flushInterval;
    this.backoffPolicy = backoffPolicy;
}

Example #26

Source File: ESBufferConfiguration.java From SkaETL with Apache License 2.0

5 votes

public BackoffPolicy toBackOffPolicy() {
    switch (backoffPolicy) {
        case NO_BACKOFF:
            return BackoffPolicy.noBackoff();
        case CONSTANT_BACKOFF:
            return BackoffPolicy.constantBackoff(TimeValue.timeValueMillis(delayBackoffTimeUnit.toMillis(delayBackoff)), backoffMaxRetry);
        case EXPONENTIAL_BACKOFF:
            return BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(delayBackoffTimeUnit.toMillis(delayBackoff)), backoffMaxRetry);
        default:
            throw new IllegalArgumentException(backoffPolicy + " not supported");
    }
}

Example #27

Source File: BulkProcessingOptions.java From ElasticUtils with MIT License

4 votes

public BackoffPolicy getBackoffPolicy() {
    return backoffPolicy;
}

Example #28

Source File: BulkProcessingOptionsBuilder.java From ElasticUtils with MIT License

4 votes

public BulkProcessingOptionsBuilder setBackoffPolicy(BackoffPolicy backoffPolicy) {
    this.backoffPolicy = backoffPolicy;
    return this;
}

Example #29

Source File: BulkProcessingOptions.java From ElasticUtils with MIT License

4 votes

public BackoffPolicy getBackoffPolicy() {
    return backoffPolicy;
}

Example #30

Source File: BulkProcessingOptionsBuilder.java From ElasticUtils with MIT License

4 votes

public BulkProcessingOptionsBuilder setBackoffPolicy(BackoffPolicy backoffPolicy) {
    this.backoffPolicy = backoffPolicy;
    return this;
}