org.elasticsearch.common.unit.ByteSizeValue Java Examples

The following examples show how to use org.elasticsearch.common.unit.ByteSizeValue. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BlobStoreIndexShardSnapshot.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * Constructs a new instance of file info
 *
 * @param name         file name as stored in the blob store
 * @param metaData  the files meta data
 * @param partSize     size of the single chunk
 */
public FileInfo(String name, StoreFileMetaData metaData, ByteSizeValue partSize) {
    this.name = name;
    this.metadata = metaData;

    long partBytes = Long.MAX_VALUE;
    if (partSize != null) {
        partBytes = partSize.bytes();
    }

    long totalLength = metaData.length();
    long numberOfParts = totalLength / partBytes;
    if (totalLength % partBytes > 0) {
        numberOfParts++;
    }
    if (numberOfParts == 0) {
        numberOfParts++;
    }
    this.numberOfParts = numberOfParts;
    this.partSize = partSize;
    this.partBytes = partBytes;
}
 
Example #2
Source File: CachedFeatureStoreTests.java    From elasticsearch-learning-to-rank with Apache License 2.0 6 votes vote down vote up
@BadApple(bugUrl = "https://github.com/o19s/elasticsearch-learning-to-rank/issues/75")
public void testExpirationOnGet() throws IOException, InterruptedException {
    Caches caches = new Caches(TimeValue.timeValueHours(1), TimeValue.timeValueMillis(100), new ByteSizeValue(1000000));
    CachedFeatureStore store = new CachedFeatureStore(memStore, caches);
    CompiledLtrModel model = LtrTestUtils.buildRandomModel();
    memStore.add(model);
    store.loadModel(model.name()); // fill cache
    store.loadModel(model.name()); // access cache
    assertNotNull(store.getCachedModel(model.name()));
    Thread.sleep(500);
    caches.modelCache().refresh();
    assertNull(store.getCachedModel(model.name()));
    assertNull(store.getCachedModel(model.name()));
    assertTrue(caches.getCachedStoreNames().isEmpty());
    assertEquals(0, caches.getPerStoreStats(memStore.getStoreName()).modelRam());
    assertEquals(0, caches.getPerStoreStats(memStore.getStoreName()).totalCount());
}
 
Example #3
Source File: ESIntegTestCase.java    From crate with Apache License 2.0 6 votes vote down vote up
private static Settings.Builder setRandomIndexTranslogSettings(Random random, Settings.Builder builder) {
    if (random.nextBoolean()) {
        builder.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
                new ByteSizeValue(RandomNumbers.randomIntBetween(random, 1, 300), ByteSizeUnit.MB));
    }
    if (random.nextBoolean()) {
        builder.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
                new ByteSizeValue(1, ByteSizeUnit.PB)); // just don't flush
    }
    if (random.nextBoolean()) {
        builder.put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(),
                RandomPicks.randomFrom(random, Translog.Durability.values()));
    }

    if (random.nextBoolean()) {
        builder.put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(),
                RandomNumbers.randomIntBetween(random, 100, 5000), TimeUnit.MILLISECONDS);
    }

    return builder;
}
 
Example #4
Source File: DiskThresholdSettings.java    From crate with Apache License 2.0 6 votes vote down vote up
private static void doValidateAsBytes(final String low, final String high, final String flood) {
    final ByteSizeValue lowWatermarkBytes =
            thresholdBytesFromWatermark(low, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), false);
    final ByteSizeValue highWatermarkBytes =
            thresholdBytesFromWatermark(high, CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), false);
    final ByteSizeValue floodStageBytes =
            thresholdBytesFromWatermark(flood, CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), false);
    if (lowWatermarkBytes.getBytes() < highWatermarkBytes.getBytes()) {
        throw new IllegalArgumentException(
                "low disk watermark [" + low + "] less than high disk watermark [" + high + "]");
    }
    if (highWatermarkBytes.getBytes() < floodStageBytes.getBytes()) {
        throw new IllegalArgumentException(
                "high disk watermark [" + high + "] less than flood stage disk watermark [" + flood + "]");
    }
}
 
Example #5
Source File: MemoryCircuitBreaker.java    From crate with Apache License 2.0 6 votes vote down vote up
/**
 * Create a circuit breaker that will break if the number of estimated
 * bytes grows above the limit. All estimations will be multiplied by
 * the given overheadConstant. Uses the given oldBreaker to initialize
 * the starting offset.
 * @param limit circuit breaker limit
 * @param overheadConstant constant multiplier for byte estimations
 * @param oldBreaker the previous circuit breaker to inherit the used value from (starting offset)
 */
public MemoryCircuitBreaker(ByteSizeValue limit, double overheadConstant, MemoryCircuitBreaker oldBreaker, Logger logger) {
    this.memoryBytesLimit = limit.getBytes();
    this.overheadConstant = overheadConstant;
    if (oldBreaker == null) {
        this.used = new AtomicLong(0);
        this.trippedCount = new AtomicLong(0);
    } else {
        this.used = oldBreaker.used;
        this.trippedCount = oldBreaker.trippedCount;
    }
    this.logger = logger;
    if (logger.isTraceEnabled()) {
        logger.trace("Creating MemoryCircuitBreaker with a limit of {} bytes ({}) and a overhead constant of {}",
                this.memoryBytesLimit, limit, this.overheadConstant);
    }
}
 
Example #6
Source File: TranslogService.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Inject
public TranslogService(ShardId shardId, IndexSettingsService indexSettingsService, ThreadPool threadPool, IndexShard indexShard) {
    super(shardId, indexSettingsService.getSettings());
    this.threadPool = threadPool;
    this.indexSettingsService = indexSettingsService;
    this.indexShard = indexShard;
    this.flushThresholdOperations = indexSettings.getAsInt(INDEX_TRANSLOG_FLUSH_THRESHOLD_OPS, indexSettings.getAsInt("index.translog.flush_threshold", 50000));
    this.flushThresholdSize = indexSettings.getAsBytesSize(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(100, ByteSizeUnit.MB));
    this.flushThresholdPeriod = indexSettings.getAsTime(INDEX_TRANSLOG_FLUSH_THRESHOLD_PERIOD, TimeValue.timeValueMinutes(10));
    this.interval = indexSettings.getAsTime(INDEX_TRANSLOG_FLUSH_INTERVAL, timeValueMillis(5000));
    this.disableFlush = indexSettings.getAsBoolean(INDEX_TRANSLOG_DISABLE_FLUSH, false);
    logger.debug("interval [{}], flush_threshold_ops [{}], flush_threshold_size [{}], flush_threshold_period [{}]", interval, flushThresholdOperations, flushThresholdSize, flushThresholdPeriod);

    this.future = threadPool.schedule(interval, ThreadPool.Names.SAME, new TranslogBasedFlush());

    indexSettingsService.addListener(applySettings);
}
 
Example #7
Source File: NodeFetchResponseTest.java    From crate with Apache License 2.0 6 votes vote down vote up
@Test
public void testResponseCircuitBreaker() throws Exception {
    NodeFetchResponse orig = new NodeFetchResponse(fetched);
    BytesStreamOutput out = new BytesStreamOutput();
    orig.writeTo(out);
    StreamInput in = out.bytes().streamInput();

    expectedException.expect(CircuitBreakingException.class);
    new NodeFetchResponse(
        in,
        streamers,
        ConcurrentRamAccounting.forCircuitBreaker(
            "test",
            new MemoryCircuitBreaker(
                new ByteSizeValue(2, ByteSizeUnit.BYTES),
                1.0,
                LogManager.getLogger(NodeFetchResponseTest.class))));

}
 
Example #8
Source File: CommonStats.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * Utility method which computes total memory by adding
 * FieldData, Percolate, Segments (memory, index writer, version map)
 */
public ByteSizeValue getTotalMemory() {
    long size = 0;
    if (this.getFieldData() != null) {
        size += this.getFieldData().getMemorySizeInBytes();
    }
    if (this.getQueryCache() != null) {
        size += this.getQueryCache().getMemorySizeInBytes();
    }
    if (this.getPercolate() != null) {
        size += this.getPercolate().getMemorySizeInBytes();
    }
    if (this.getSegments() != null) {
        size += this.getSegments().getMemoryInBytes() +
                this.getSegments().getIndexWriterMemoryInBytes() +
                this.getSegments().getVersionMapMemoryInBytes();
    }

    return new ByteSizeValue(size);
}
 
Example #9
Source File: RowAccountingWithEstimatorsTest.java    From crate with Apache License 2.0 5 votes vote down vote up
@Test
public void testRowCellsAccountingCircuitBreakingWorks() throws Exception {
    RowCellsAccountingWithEstimators rowAccounting = new RowCellsAccountingWithEstimators(
        Collections.singletonList(DataTypes.INTEGER),
        ConcurrentRamAccounting.forCircuitBreaker(
            "test",
            new MemoryCircuitBreaker(
                new ByteSizeValue(10, ByteSizeUnit.BYTES), 1.01, LogManager.getLogger(RowAccountingWithEstimatorsTest.class))
        ), 0);

    expectedException.expect(CircuitBreakingException.class);
    IntStream.range(0, 3).forEach(i -> rowAccounting.accountForAndMaybeBreak(new Object[]{i}));
}
 
Example #10
Source File: XmlXContentBuilder.java    From elasticsearch-xml with Apache License 2.0 5 votes vote down vote up
public XmlXContentBuilder byteSizeField(XContentBuilderString rawFieldName, XContentBuilderString readableFieldName, long rawSize) throws IOException {
    if (humanReadable) {
        field(readableFieldName, new ByteSizeValue(rawSize).toString());
    }
    field(rawFieldName, rawSize);
    return this;
}
 
Example #11
Source File: Caches.java    From elasticsearch-learning-to-rank with Apache License 2.0 5 votes vote down vote up
public Caches(TimeValue expAfterWrite, TimeValue expAfterAccess, ByteSizeValue maxWeight) {
    this.featureCache = configCache(CacheBuilder.<CacheKey, Feature>builder(), expAfterWrite, expAfterAccess, maxWeight)
            .weigher(Caches::weigther)
            .removalListener((l) -> this.onRemove(l.getKey(), l.getValue()))
            .build();
    this.featureSetCache = configCache(CacheBuilder.<CacheKey, FeatureSet>builder(), expAfterWrite, expAfterAccess, maxWeight)
            .weigher(Caches::weigther)
            .removalListener((l) -> this.onRemove(l.getKey(), l.getValue()))
            .build();
    this.modelCache = configCache(CacheBuilder.<CacheKey, CompiledLtrModel>builder(), expAfterWrite, expAfterAccess, maxWeight)
            .weigher((s, w) -> w.ramBytesUsed())
            .removalListener((l) -> this.onRemove(l.getKey(), l.getValue()))
            .build();
    this.maxWeight = maxWeight.getBytes();
}
 
Example #12
Source File: BulkProcessingOptions.java    From ElasticUtils with MIT License 5 votes vote down vote up
public BulkProcessingOptions(String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval, BackoffPolicy backoffPolicy) {
    this.name = name;
    this.concurrentRequests = concurrentRequests;
    this.bulkActions = bulkActions;
    this.bulkSize = bulkSize;
    this.flushInterval = flushInterval;
    this.backoffPolicy = backoffPolicy;
}
 
Example #13
Source File: HierarchyCircuitBreakerService.java    From crate with Apache License 2.0 5 votes vote down vote up
public static String breakingExceptionMessage(String label, long limit) {
    return String.format(
        Locale.ENGLISH,
        BREAKING_EXCEPTION_MESSAGE,
        label,
        limit,
        new ByteSizeValue(limit)
    );
}
 
Example #14
Source File: ElasticsearchSinkBase.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Build the {@link BulkProcessor}.
 *
 * <p>Note: this is exposed for testing purposes.
 */
@VisibleForTesting
protected BulkProcessor buildBulkProcessor(BulkProcessor.Listener listener) {
	checkNotNull(listener);

	BulkProcessor.Builder bulkProcessorBuilder = callBridge.createBulkProcessorBuilder(client, listener);

	// This makes flush() blocking
	bulkProcessorBuilder.setConcurrentRequests(0);

	if (bulkProcessorFlushMaxActions != null) {
		bulkProcessorBuilder.setBulkActions(bulkProcessorFlushMaxActions);
	}

	if (bulkProcessorFlushMaxSizeMb != null) {
		bulkProcessorBuilder.setBulkSize(new ByteSizeValue(bulkProcessorFlushMaxSizeMb, ByteSizeUnit.MB));
	}

	if (bulkProcessorFlushIntervalMillis != null) {
		bulkProcessorBuilder.setFlushInterval(TimeValue.timeValueMillis(bulkProcessorFlushIntervalMillis));
	}

	// if backoff retrying is disabled, bulkProcessorFlushBackoffPolicy will be null
	callBridge.configureBulkProcessorBackoff(bulkProcessorBuilder, bulkProcessorFlushBackoffPolicy);

	return bulkProcessorBuilder.build();
}
 
Example #15
Source File: ChildMemoryCircuitBreaker.java    From crate with Apache License 2.0 5 votes vote down vote up
private long limit(long bytes, String label) {
    long newUsed;// Otherwise, check the addition and commit the addition, looping if
    // there are conflicts. May result in additional logging, but it's
    // trace logging and shouldn't be counted on for additions.
    long currentUsed;
    do {
        currentUsed = this.used.get();
        newUsed = currentUsed + bytes;
        long newUsedWithOverhead = (long) (newUsed * overheadConstant);
        if (logger.isTraceEnabled()) {
            logger.trace("[{}] Adding [{}][{}] to used bytes [new used: [{}], limit: {} [{}], estimate: {} [{}]]",
                    this.name,
                    new ByteSizeValue(bytes), label, new ByteSizeValue(newUsed),
                    memoryBytesLimit, new ByteSizeValue(memoryBytesLimit),
                    newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead));
        }
        if (memoryBytesLimit > 0 && newUsedWithOverhead > memoryBytesLimit) {
            logger.warn("[{}] New used memory {} [{}] for data of [{}] would be larger than configured breaker: {} [{}], breaking",
                    this.name,
                    newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead), label,
                    memoryBytesLimit, new ByteSizeValue(memoryBytesLimit));
            circuitBreak(label, newUsedWithOverhead);
        }
        // Attempt to set the new used value, but make sure it hasn't changed
        // underneath us, if it has, keep trying until we are able to set it
    } while (!this.used.compareAndSet(currentUsed, newUsed));
    return newUsed;
}
 
Example #16
Source File: RecoveryState.java    From crate with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
    // stream size first, as it matters more and the files section can be long
    builder.startObject(Fields.SIZE);
    builder.humanReadableField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, new ByteSizeValue(totalBytes()));
    builder.humanReadableField(Fields.REUSED_IN_BYTES, Fields.REUSED, new ByteSizeValue(reusedBytes()));
    builder.humanReadableField(Fields.RECOVERED_IN_BYTES, Fields.RECOVERED, new ByteSizeValue(recoveredBytes()));
    builder.field(Fields.PERCENT, String.format(Locale.ROOT, "%1.1f%%", recoveredBytesPercent()));
    builder.endObject();

    builder.startObject(Fields.FILES);
    builder.field(Fields.TOTAL, totalFileCount());
    builder.field(Fields.REUSED, reusedFileCount());
    builder.field(Fields.RECOVERED, recoveredFileCount());
    builder.field(Fields.PERCENT, String.format(Locale.ROOT, "%1.1f%%", recoveredFilesPercent()));
    if (params.paramAsBoolean("details", false)) {
        builder.startArray(Fields.DETAILS);
        for (File file : fileDetails.values()) {
            file.toXContent(builder, params);
        }
        builder.endArray();
    }
    builder.endObject();
    builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time()));
    builder.humanReadableField(Fields.SOURCE_THROTTLE_TIME_IN_MILLIS, Fields.SOURCE_THROTTLE_TIME, sourceThrottling());
    builder.humanReadableField(Fields.TARGET_THROTTLE_TIME_IN_MILLIS, Fields.TARGET_THROTTLE_TIME, targetThrottling());
    return builder;
}
 
Example #17
Source File: BulkWriterCollector.java    From elasticsearch-inout-plugin with Apache License 2.0 5 votes vote down vote up
@Override
public void open() throws WriterException {

    bulkListener = new BulkListener();
    bulkProcessor = BulkProcessor.builder(getClient(),
            bulkListener).setBulkActions(1000).setBulkSize(
            new ByteSizeValue(5, ByteSizeUnit.MB)).setFlushInterval(
            TimeValue.timeValueSeconds(5)).setConcurrentRequests(
            1).build();
}
 
Example #18
Source File: XmlXContentBuilder.java    From elasticsearch-xml with Apache License 2.0 5 votes vote down vote up
public XmlXContentBuilder byteSizeField(XContentBuilderString rawFieldName, XContentBuilderString readableFieldName, ByteSizeValue byteSizeValue) throws IOException {
    if (humanReadable) {
        field(readableFieldName, byteSizeValue.toString());
    }
    field(rawFieldName, byteSizeValue.bytes());
    return this;
}
 
Example #19
Source File: CircuitBreakerStats.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
    builder.startObject(name.toLowerCase(Locale.ROOT));
    builder.field(Fields.LIMIT, limit);
    builder.field(Fields.LIMIT_HUMAN, new ByteSizeValue(limit));
    builder.field(Fields.ESTIMATED, estimated);
    builder.field(Fields.ESTIMATED_HUMAN, new ByteSizeValue(estimated));
    builder.field(Fields.OVERHEAD, overhead);
    builder.field(Fields.TRIPPED_COUNT, trippedCount);
    builder.endObject();
    return builder;
}
 
Example #20
Source File: CircuitBreakerStats.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public String toString() {
    return "[" + this.name +
            ",limit=" + this.limit + "/" + new ByteSizeValue(this.limit) +
            ",estimated=" + this.estimated + "/" + new ByteSizeValue(this.estimated) +
            ",overhead=" + this.overhead + ",tripped=" + this.trippedCount + "]";
}
 
Example #21
Source File: MockTcpTransport.java    From crate with Apache License 2.0 5 votes vote down vote up
private void configureSocket(Socket socket) throws SocketException {
    socket.setTcpNoDelay(TransportSettings.TCP_NO_DELAY.get(settings));
    ByteSizeValue tcpSendBufferSize = TransportSettings.TCP_SEND_BUFFER_SIZE.get(settings);
    if (tcpSendBufferSize.getBytes() > 0) {
        socket.setSendBufferSize(tcpSendBufferSize.bytesAsInt());
    }
    ByteSizeValue tcpReceiveBufferSize = TransportSettings.TCP_RECEIVE_BUFFER_SIZE.get(settings);
    if (tcpReceiveBufferSize.getBytes() > 0) {
        socket.setReceiveBufferSize(tcpReceiveBufferSize.bytesAsInt());
    }
    socket.setReuseAddress(TransportSettings.TCP_REUSE_ADDRESS.get(settings));
}
 
Example #22
Source File: ChildMemoryCircuitBreaker.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Method used to trip the breaker, delegates to the parent to determine
 * whether to trip the breaker or not
 */
private void circuitBreak(String fieldName, long bytesNeeded) {
    this.trippedCount.incrementAndGet();
    final String message = "[" + this.name + "] Data too large, data for [" + fieldName + "]" +
            " would be [" + bytesNeeded + "/" + new ByteSizeValue(bytesNeeded) + "]" +
            ", which is larger than the limit of [" +
            memoryBytesLimit + "/" + new ByteSizeValue(memoryBytesLimit) + "]";
    logger.debug("{}", message);
    throw new CircuitBreakingException(message, bytesNeeded, memoryBytesLimit);
}
 
Example #23
Source File: DiskUsage.java    From crate with Apache License 2.0 5 votes vote down vote up
XContentBuilder toShortXContent(XContentBuilder builder) throws IOException {
    builder.field("path", this.path);
    builder.humanReadableField("total_bytes", "total", new ByteSizeValue(this.totalBytes));
    builder.humanReadableField("used_bytes", "used", new ByteSizeValue(this.getUsedBytes()));
    builder.humanReadableField("free_bytes", "free", new ByteSizeValue(this.freeBytes));
    builder.field("free_disk_percent", truncatePercent(this.getFreeDiskAsPercentage()));
    builder.field("used_disk_percent", truncatePercent(this.getUsedDiskAsPercentage()));
    return builder;
}
 
Example #24
Source File: BulkProcessingOptionsBuilderTest.java    From ElasticUtils with MIT License 5 votes vote down vote up
@Test
public void default_values_set_when_initializing_builder() {

    // Build:
    BulkProcessingOptions options = new BulkProcessingOptionsBuilder().build();

    // Check Values:
    Assert.assertEquals(null, options.getName());
    Assert.assertEquals(1, options.getConcurrentRequests());
    Assert.assertEquals(1000, options.getBulkActions());
    Assert.assertEquals(new ByteSizeValue(5, ByteSizeUnit.MB).bytes(), options.getBulkSize().bytes());
    Assert.assertEquals(null, options.getFlushInterval());
    Assert.assertEquals(BackoffPolicy.exponentialBackoff().getClass(), options.getBackoffPolicy().getClass());
}
 
Example #25
Source File: SettingTests.java    From crate with Apache License 2.0 5 votes vote down vote up
public void testMemorySize() {
    Setting<ByteSizeValue> memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", new ByteSizeValue(1024), Property.Dynamic,
            Property.NodeScope);

    assertFalse(memorySizeValueSetting.isGroupSetting());
    ByteSizeValue memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
    assertEquals(memorySizeValue.getBytes(), 1024);

    memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", s -> "2048b", Property.Dynamic, Property.NodeScope);
    memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
    assertEquals(memorySizeValue.getBytes(), 2048);

    memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", "50%", Property.Dynamic, Property.NodeScope);
    assertFalse(memorySizeValueSetting.isGroupSetting());
    memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
    assertEquals(memorySizeValue.getBytes(), JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.5, 1.0);

    memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", s -> "25%", Property.Dynamic, Property.NodeScope);
    memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
    assertEquals(memorySizeValue.getBytes(), JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.25, 1.0);

    AtomicReference<ByteSizeValue> value = new AtomicReference<>(null);
    ClusterSettings.SettingUpdater<ByteSizeValue> settingUpdater = memorySizeValueSetting.newUpdater(value::set, logger);

    assertTrue(settingUpdater.apply(Settings.builder().put("a.byte.size", "12").build(), Settings.EMPTY));
    assertEquals(new ByteSizeValue(12), value.get());

    assertTrue(settingUpdater.apply(Settings.builder().put("a.byte.size", "12b").build(), Settings.EMPTY));
    assertEquals(new ByteSizeValue(12), value.get());

    assertTrue(settingUpdater.apply(Settings.builder().put("a.byte.size", "20%").build(), Settings.EMPTY));
    assertEquals(new ByteSizeValue((int) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.2)), value.get());
}
 
Example #26
Source File: MemorySizeSettingsTests.java    From crate with Apache License 2.0 5 votes vote down vote up
private void assertMemorySizeSetting(Setting<ByteSizeValue> setting, String settingKey, ByteSizeValue defaultValue) {
    assertThat(setting, notNullValue());
    assertThat(setting.getKey(), equalTo(settingKey));
    assertThat(setting.getProperties(), hasItem(Property.NodeScope));
    assertThat(setting.getDefault(Settings.EMPTY),
            equalTo(defaultValue));
    Settings settingWithPercentage = Settings.builder().put(settingKey, "25%").build();
    assertThat(setting.get(settingWithPercentage),
            equalTo(new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.25))));
    Settings settingWithBytesValue = Settings.builder().put(settingKey, "1024b").build();
    assertThat(setting.get(settingWithBytesValue), equalTo(new ByteSizeValue(1024)));
}
 
Example #27
Source File: AbstractGatherer.java    From elasticsearch-gatherer with Apache License 2.0 5 votes vote down vote up
public Gatherer start() throws ElasticSearchException {
    //this.logger = Loggers.getLogger(getClass(), settings.globalSettings(), riverName);
    this.bulkActions = settings.getAsInt("bulk_actions", 1000);
    this.bulkSize = settings.getAsBytesSize("bulk_size", new ByteSizeValue(5, ByteSizeUnit.MB));
    this.flushInterval = settings.getAsTime("flush_interval", TimeValue.timeValueSeconds(5));
    this.concurrentRequests = settings.getAsInt("concurrent_requests", 4);
    bulkProcessor = BulkProcessor.builder(client, new BulkListener())
            .setBulkActions(bulkActions)
            .setBulkSize(bulkSize)
            .setFlushInterval(flushInterval)
            .setConcurrentRequests(concurrentRequests)
            .build();
    return this;
}
 
Example #28
Source File: TranslogWriter.java    From crate with Apache License 2.0 5 votes vote down vote up
private TranslogWriter(
    final ChannelFactory channelFactory,
    final ShardId shardId,
    final Checkpoint initialCheckpoint,
    final FileChannel channel,
    final Path path,
    final ByteSizeValue bufferSize,
    final LongSupplier globalCheckpointSupplier, LongSupplier minTranslogGenerationSupplier, TranslogHeader header,
    TragicExceptionHolder tragedy,
    final LongConsumer persistedSequenceNumberConsumer)
        throws
        IOException {
    super(initialCheckpoint.generation, channel, path, header);
    assert initialCheckpoint.offset == channel.position() :
        "initial checkpoint offset [" + initialCheckpoint.offset + "] is different than current channel position ["
            + channel.position() + "]";
    this.shardId = shardId;
    this.channelFactory = channelFactory;
    this.minTranslogGenerationSupplier = minTranslogGenerationSupplier;
    this.outputStream = new BufferedChannelOutputStream(java.nio.channels.Channels.newOutputStream(channel), bufferSize.bytesAsInt());
    this.lastSyncedCheckpoint = initialCheckpoint;
    this.totalOffset = initialCheckpoint.offset;
    assert initialCheckpoint.minSeqNo == SequenceNumbers.NO_OPS_PERFORMED : initialCheckpoint.minSeqNo;
    this.minSeqNo = initialCheckpoint.minSeqNo;
    assert initialCheckpoint.maxSeqNo == SequenceNumbers.NO_OPS_PERFORMED : initialCheckpoint.maxSeqNo;
    this.maxSeqNo = initialCheckpoint.maxSeqNo;
    assert initialCheckpoint.trimmedAboveSeqNo == SequenceNumbers.UNASSIGNED_SEQ_NO : initialCheckpoint.trimmedAboveSeqNo;
    this.globalCheckpointSupplier = globalCheckpointSupplier;
    this.nonFsyncedSequenceNumbers = new LongArrayList(64);
    this.persistedSequenceNumberConsumer = persistedSequenceNumberConsumer;
    this.seenSequenceNumbers = Assertions.ENABLED ? new HashMap<>() : null;
    this.tragedy = tragedy;
}
 
Example #29
Source File: BulkProcessingOptions.java    From ElasticUtils with MIT License 5 votes vote down vote up
public BulkProcessingOptions(String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval, BackoffPolicy backoffPolicy) {
    this.name = name;
    this.concurrentRequests = concurrentRequests;
    this.bulkActions = bulkActions;
    this.bulkSize = bulkSize;
    this.flushInterval = flushInterval;
    this.backoffPolicy = backoffPolicy;
}
 
Example #30
Source File: Segment.java    From crate with Apache License 2.0 4 votes vote down vote up
public ByteSizeValue getSize() {
    return new ByteSizeValue(sizeInBytes);
}