Java Code Examples for org.elasticsearch.common.unit.ByteSizeValue

The following examples show how to use org.elasticsearch.common.unit.ByteSizeValue. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: crate   Source File: NodeFetchResponseTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResponseCircuitBreaker() throws Exception {
    NodeFetchResponse orig = new NodeFetchResponse(fetched);
    BytesStreamOutput out = new BytesStreamOutput();
    orig.writeTo(out);
    StreamInput in = out.bytes().streamInput();

    expectedException.expect(CircuitBreakingException.class);
    new NodeFetchResponse(
        in,
        streamers,
        ConcurrentRamAccounting.forCircuitBreaker(
            "test",
            new MemoryCircuitBreaker(
                new ByteSizeValue(2, ByteSizeUnit.BYTES),
                1.0,
                LogManager.getLogger(NodeFetchResponseTest.class))));

}
 
Example 2
Source Project: crate   Source File: MemoryCircuitBreaker.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create a circuit breaker that will break if the number of estimated
 * bytes grows above the limit. All estimations will be multiplied by
 * the given overheadConstant. Uses the given oldBreaker to initialize
 * the starting offset.
 * @param limit circuit breaker limit
 * @param overheadConstant constant multiplier for byte estimations
 * @param oldBreaker the previous circuit breaker to inherit the used value from (starting offset)
 */
public MemoryCircuitBreaker(ByteSizeValue limit, double overheadConstant, MemoryCircuitBreaker oldBreaker, Logger logger) {
    this.memoryBytesLimit = limit.getBytes();
    this.overheadConstant = overheadConstant;
    if (oldBreaker == null) {
        this.used = new AtomicLong(0);
        this.trippedCount = new AtomicLong(0);
    } else {
        this.used = oldBreaker.used;
        this.trippedCount = oldBreaker.trippedCount;
    }
    this.logger = logger;
    if (logger.isTraceEnabled()) {
        logger.trace("Creating MemoryCircuitBreaker with a limit of {} bytes ({}) and a overhead constant of {}",
                this.memoryBytesLimit, limit, this.overheadConstant);
    }
}
 
Example 3
@BadApple(bugUrl = "https://github.com/o19s/elasticsearch-learning-to-rank/issues/75")
public void testExpirationOnGet() throws IOException, InterruptedException {
    Caches caches = new Caches(TimeValue.timeValueHours(1), TimeValue.timeValueMillis(100), new ByteSizeValue(1000000));
    CachedFeatureStore store = new CachedFeatureStore(memStore, caches);
    CompiledLtrModel model = LtrTestUtils.buildRandomModel();
    memStore.add(model);
    store.loadModel(model.name()); // fill cache
    store.loadModel(model.name()); // access cache
    assertNotNull(store.getCachedModel(model.name()));
    Thread.sleep(500);
    caches.modelCache().refresh();
    assertNull(store.getCachedModel(model.name()));
    assertNull(store.getCachedModel(model.name()));
    assertTrue(caches.getCachedStoreNames().isEmpty());
    assertEquals(0, caches.getPerStoreStats(memStore.getStoreName()).modelRam());
    assertEquals(0, caches.getPerStoreStats(memStore.getStoreName()).totalCount());
}
 
Example 4
Source Project: crate   Source File: ESIntegTestCase.java    License: Apache License 2.0 6 votes vote down vote up
private static Settings.Builder setRandomIndexTranslogSettings(Random random, Settings.Builder builder) {
    if (random.nextBoolean()) {
        builder.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
                new ByteSizeValue(RandomNumbers.randomIntBetween(random, 1, 300), ByteSizeUnit.MB));
    }
    if (random.nextBoolean()) {
        builder.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
                new ByteSizeValue(1, ByteSizeUnit.PB)); // just don't flush
    }
    if (random.nextBoolean()) {
        builder.put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(),
                RandomPicks.randomFrom(random, Translog.Durability.values()));
    }

    if (random.nextBoolean()) {
        builder.put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(),
                RandomNumbers.randomIntBetween(random, 100, 5000), TimeUnit.MILLISECONDS);
    }

    return builder;
}
 
Example 5
Source Project: crate   Source File: DiskThresholdSettings.java    License: Apache License 2.0 6 votes vote down vote up
private static void doValidateAsBytes(final String low, final String high, final String flood) {
    final ByteSizeValue lowWatermarkBytes =
            thresholdBytesFromWatermark(low, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), false);
    final ByteSizeValue highWatermarkBytes =
            thresholdBytesFromWatermark(high, CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), false);
    final ByteSizeValue floodStageBytes =
            thresholdBytesFromWatermark(flood, CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), false);
    if (lowWatermarkBytes.getBytes() < highWatermarkBytes.getBytes()) {
        throw new IllegalArgumentException(
                "low disk watermark [" + low + "] less than high disk watermark [" + high + "]");
    }
    if (highWatermarkBytes.getBytes() < floodStageBytes.getBytes()) {
        throw new IllegalArgumentException(
                "high disk watermark [" + high + "] less than flood stage disk watermark [" + flood + "]");
    }
}
 
Example 6
Source Project: Elasticsearch   Source File: CommonStats.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Utility method which computes total memory by adding
 * FieldData, Percolate, Segments (memory, index writer, version map)
 */
public ByteSizeValue getTotalMemory() {
    long size = 0;
    if (this.getFieldData() != null) {
        size += this.getFieldData().getMemorySizeInBytes();
    }
    if (this.getQueryCache() != null) {
        size += this.getQueryCache().getMemorySizeInBytes();
    }
    if (this.getPercolate() != null) {
        size += this.getPercolate().getMemorySizeInBytes();
    }
    if (this.getSegments() != null) {
        size += this.getSegments().getMemoryInBytes() +
                this.getSegments().getIndexWriterMemoryInBytes() +
                this.getSegments().getVersionMapMemoryInBytes();
    }

    return new ByteSizeValue(size);
}
 
Example 7
Source Project: Elasticsearch   Source File: TranslogService.java    License: Apache License 2.0 6 votes vote down vote up
@Inject
public TranslogService(ShardId shardId, IndexSettingsService indexSettingsService, ThreadPool threadPool, IndexShard indexShard) {
    super(shardId, indexSettingsService.getSettings());
    this.threadPool = threadPool;
    this.indexSettingsService = indexSettingsService;
    this.indexShard = indexShard;
    this.flushThresholdOperations = indexSettings.getAsInt(INDEX_TRANSLOG_FLUSH_THRESHOLD_OPS, indexSettings.getAsInt("index.translog.flush_threshold", 50000));
    this.flushThresholdSize = indexSettings.getAsBytesSize(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(100, ByteSizeUnit.MB));
    this.flushThresholdPeriod = indexSettings.getAsTime(INDEX_TRANSLOG_FLUSH_THRESHOLD_PERIOD, TimeValue.timeValueMinutes(10));
    this.interval = indexSettings.getAsTime(INDEX_TRANSLOG_FLUSH_INTERVAL, timeValueMillis(5000));
    this.disableFlush = indexSettings.getAsBoolean(INDEX_TRANSLOG_DISABLE_FLUSH, false);
    logger.debug("interval [{}], flush_threshold_ops [{}], flush_threshold_size [{}], flush_threshold_period [{}]", interval, flushThresholdOperations, flushThresholdSize, flushThresholdPeriod);

    this.future = threadPool.schedule(interval, ThreadPool.Names.SAME, new TranslogBasedFlush());

    indexSettingsService.addListener(applySettings);
}
 
Example 8
Source Project: Elasticsearch   Source File: BlobStoreIndexShardSnapshot.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Constructs a new instance of file info
 *
 * @param name         file name as stored in the blob store
 * @param metaData  the files meta data
 * @param partSize     size of the single chunk
 */
public FileInfo(String name, StoreFileMetaData metaData, ByteSizeValue partSize) {
    this.name = name;
    this.metadata = metaData;

    long partBytes = Long.MAX_VALUE;
    if (partSize != null) {
        partBytes = partSize.bytes();
    }

    long totalLength = metaData.length();
    long numberOfParts = totalLength / partBytes;
    if (totalLength % partBytes > 0) {
        numberOfParts++;
    }
    if (numberOfParts == 0) {
        numberOfParts++;
    }
    this.numberOfParts = numberOfParts;
    this.partSize = partSize;
    this.partBytes = partBytes;
}
 
Example 9
Source Project: ElasticUtils   Source File: BulkProcessingOptions.java    License: MIT License 5 votes vote down vote up
public BulkProcessingOptions(String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval, BackoffPolicy backoffPolicy) {
    this.name = name;
    this.concurrentRequests = concurrentRequests;
    this.bulkActions = bulkActions;
    this.bulkSize = bulkSize;
    this.flushInterval = flushInterval;
    this.backoffPolicy = backoffPolicy;
}
 
Example 10
Source Project: crate   Source File: RecoveryState.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
    // stream size first, as it matters more and the files section can be long
    builder.startObject(Fields.SIZE);
    builder.humanReadableField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, new ByteSizeValue(totalBytes()));
    builder.humanReadableField(Fields.REUSED_IN_BYTES, Fields.REUSED, new ByteSizeValue(reusedBytes()));
    builder.humanReadableField(Fields.RECOVERED_IN_BYTES, Fields.RECOVERED, new ByteSizeValue(recoveredBytes()));
    builder.field(Fields.PERCENT, String.format(Locale.ROOT, "%1.1f%%", recoveredBytesPercent()));
    builder.endObject();

    builder.startObject(Fields.FILES);
    builder.field(Fields.TOTAL, totalFileCount());
    builder.field(Fields.REUSED, reusedFileCount());
    builder.field(Fields.RECOVERED, recoveredFileCount());
    builder.field(Fields.PERCENT, String.format(Locale.ROOT, "%1.1f%%", recoveredFilesPercent()));
    if (params.paramAsBoolean("details", false)) {
        builder.startArray(Fields.DETAILS);
        for (File file : fileDetails.values()) {
            file.toXContent(builder, params);
        }
        builder.endArray();
    }
    builder.endObject();
    builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time()));
    builder.humanReadableField(Fields.SOURCE_THROTTLE_TIME_IN_MILLIS, Fields.SOURCE_THROTTLE_TIME, sourceThrottling());
    builder.humanReadableField(Fields.TARGET_THROTTLE_TIME_IN_MILLIS, Fields.TARGET_THROTTLE_TIME, targetThrottling());
    return builder;
}
 
Example 11
Source Project: ElasticUtils   Source File: BulkProcessingOptions.java    License: MIT License 5 votes vote down vote up
public BulkProcessingOptions(String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval, BackoffPolicy backoffPolicy) {
    this.name = name;
    this.concurrentRequests = concurrentRequests;
    this.bulkActions = bulkActions;
    this.bulkSize = bulkSize;
    this.flushInterval = flushInterval;
    this.backoffPolicy = backoffPolicy;
}
 
Example 12
Source Project: elasticsearch-learning-to-rank   Source File: Caches.java    License: Apache License 2.0 5 votes vote down vote up
public Caches(TimeValue expAfterWrite, TimeValue expAfterAccess, ByteSizeValue maxWeight) {
    this.featureCache = configCache(CacheBuilder.<CacheKey, Feature>builder(), expAfterWrite, expAfterAccess, maxWeight)
            .weigher(Caches::weigther)
            .removalListener((l) -> this.onRemove(l.getKey(), l.getValue()))
            .build();
    this.featureSetCache = configCache(CacheBuilder.<CacheKey, FeatureSet>builder(), expAfterWrite, expAfterAccess, maxWeight)
            .weigher(Caches::weigther)
            .removalListener((l) -> this.onRemove(l.getKey(), l.getValue()))
            .build();
    this.modelCache = configCache(CacheBuilder.<CacheKey, CompiledLtrModel>builder(), expAfterWrite, expAfterAccess, maxWeight)
            .weigher((s, w) -> w.ramBytesUsed())
            .removalListener((l) -> this.onRemove(l.getKey(), l.getValue()))
            .build();
    this.maxWeight = maxWeight.getBytes();
}
 
Example 13
Source Project: crate   Source File: ChildMemoryCircuitBreaker.java    License: Apache License 2.0 5 votes vote down vote up
private long limit(long bytes, String label) {
    long newUsed;// Otherwise, check the addition and commit the addition, looping if
    // there are conflicts. May result in additional logging, but it's
    // trace logging and shouldn't be counted on for additions.
    long currentUsed;
    do {
        currentUsed = this.used.get();
        newUsed = currentUsed + bytes;
        long newUsedWithOverhead = (long) (newUsed * overheadConstant);
        if (logger.isTraceEnabled()) {
            logger.trace("[{}] Adding [{}][{}] to used bytes [new used: [{}], limit: {} [{}], estimate: {} [{}]]",
                    this.name,
                    new ByteSizeValue(bytes), label, new ByteSizeValue(newUsed),
                    memoryBytesLimit, new ByteSizeValue(memoryBytesLimit),
                    newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead));
        }
        if (memoryBytesLimit > 0 && newUsedWithOverhead > memoryBytesLimit) {
            logger.warn("[{}] New used memory {} [{}] for data of [{}] would be larger than configured breaker: {} [{}], breaking",
                    this.name,
                    newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead), label,
                    memoryBytesLimit, new ByteSizeValue(memoryBytesLimit));
            circuitBreak(label, newUsedWithOverhead);
        }
        // Attempt to set the new used value, but make sure it hasn't changed
        // underneath us, if it has, keep trying until we are able to set it
    } while (!this.used.compareAndSet(currentUsed, newUsed));
    return newUsed;
}
 
Example 14
Source Project: flink   Source File: ElasticsearchSinkBase.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Build the {@link BulkProcessor}.
 *
 * <p>Note: this is exposed for testing purposes.
 */
@VisibleForTesting
protected BulkProcessor buildBulkProcessor(BulkProcessor.Listener listener) {
	checkNotNull(listener);

	BulkProcessor.Builder bulkProcessorBuilder = callBridge.createBulkProcessorBuilder(client, listener);

	// This makes flush() blocking
	bulkProcessorBuilder.setConcurrentRequests(0);

	if (bulkProcessorFlushMaxActions != null) {
		bulkProcessorBuilder.setBulkActions(bulkProcessorFlushMaxActions);
	}

	if (bulkProcessorFlushMaxSizeMb != null) {
		bulkProcessorBuilder.setBulkSize(new ByteSizeValue(bulkProcessorFlushMaxSizeMb, ByteSizeUnit.MB));
	}

	if (bulkProcessorFlushIntervalMillis != null) {
		bulkProcessorBuilder.setFlushInterval(TimeValue.timeValueMillis(bulkProcessorFlushIntervalMillis));
	}

	// if backoff retrying is disabled, bulkProcessorFlushBackoffPolicy will be null
	callBridge.configureBulkProcessorBackoff(bulkProcessorBuilder, bulkProcessorFlushBackoffPolicy);

	return bulkProcessorBuilder.build();
}
 
Example 15
Source Project: Elasticsearch   Source File: CircuitBreakerStats.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
    builder.startObject(name.toLowerCase(Locale.ROOT));
    builder.field(Fields.LIMIT, limit);
    builder.field(Fields.LIMIT_HUMAN, new ByteSizeValue(limit));
    builder.field(Fields.ESTIMATED, estimated);
    builder.field(Fields.ESTIMATED_HUMAN, new ByteSizeValue(estimated));
    builder.field(Fields.OVERHEAD, overhead);
    builder.field(Fields.TRIPPED_COUNT, trippedCount);
    builder.endObject();
    return builder;
}
 
Example 16
Source Project: Elasticsearch   Source File: CircuitBreakerStats.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public String toString() {
    return "[" + this.name +
            ",limit=" + this.limit + "/" + new ByteSizeValue(this.limit) +
            ",estimated=" + this.estimated + "/" + new ByteSizeValue(this.estimated) +
            ",overhead=" + this.overhead + ",tripped=" + this.trippedCount + "]";
}
 
Example 17
Source Project: crate   Source File: MockTcpTransport.java    License: Apache License 2.0 5 votes vote down vote up
private void configureSocket(Socket socket) throws SocketException {
    socket.setTcpNoDelay(TransportSettings.TCP_NO_DELAY.get(settings));
    ByteSizeValue tcpSendBufferSize = TransportSettings.TCP_SEND_BUFFER_SIZE.get(settings);
    if (tcpSendBufferSize.getBytes() > 0) {
        socket.setSendBufferSize(tcpSendBufferSize.bytesAsInt());
    }
    ByteSizeValue tcpReceiveBufferSize = TransportSettings.TCP_RECEIVE_BUFFER_SIZE.get(settings);
    if (tcpReceiveBufferSize.getBytes() > 0) {
        socket.setReceiveBufferSize(tcpReceiveBufferSize.bytesAsInt());
    }
    socket.setReuseAddress(TransportSettings.TCP_REUSE_ADDRESS.get(settings));
}
 
Example 18
Source Project: crate   Source File: DiskUsage.java    License: Apache License 2.0 5 votes vote down vote up
XContentBuilder toShortXContent(XContentBuilder builder) throws IOException {
    builder.field("path", this.path);
    builder.humanReadableField("total_bytes", "total", new ByteSizeValue(this.totalBytes));
    builder.humanReadableField("used_bytes", "used", new ByteSizeValue(this.getUsedBytes()));
    builder.humanReadableField("free_bytes", "free", new ByteSizeValue(this.freeBytes));
    builder.field("free_disk_percent", truncatePercent(this.getFreeDiskAsPercentage()));
    builder.field("used_disk_percent", truncatePercent(this.getUsedDiskAsPercentage()));
    return builder;
}
 
Example 19
Source Project: ElasticUtils   Source File: BulkProcessingOptionsBuilderTest.java    License: MIT License 5 votes vote down vote up
@Test
public void default_values_set_when_initializing_builder() {

    // Build:
    BulkProcessingOptions options = new BulkProcessingOptionsBuilder().build();

    // Check Values:
    Assert.assertEquals(null, options.getName());
    Assert.assertEquals(1, options.getConcurrentRequests());
    Assert.assertEquals(1000, options.getBulkActions());
    Assert.assertEquals(new ByteSizeValue(5, ByteSizeUnit.MB).bytes(), options.getBulkSize().bytes());
    Assert.assertEquals(null, options.getFlushInterval());
    Assert.assertEquals(BackoffPolicy.exponentialBackoff().getClass(), options.getBackoffPolicy().getClass());
}
 
Example 20
Source Project: crate   Source File: SettingTests.java    License: Apache License 2.0 5 votes vote down vote up
public void testMemorySize() {
    Setting<ByteSizeValue> memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", new ByteSizeValue(1024), Property.Dynamic,
            Property.NodeScope);

    assertFalse(memorySizeValueSetting.isGroupSetting());
    ByteSizeValue memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
    assertEquals(memorySizeValue.getBytes(), 1024);

    memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", s -> "2048b", Property.Dynamic, Property.NodeScope);
    memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
    assertEquals(memorySizeValue.getBytes(), 2048);

    memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", "50%", Property.Dynamic, Property.NodeScope);
    assertFalse(memorySizeValueSetting.isGroupSetting());
    memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
    assertEquals(memorySizeValue.getBytes(), JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.5, 1.0);

    memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", s -> "25%", Property.Dynamic, Property.NodeScope);
    memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
    assertEquals(memorySizeValue.getBytes(), JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.25, 1.0);

    AtomicReference<ByteSizeValue> value = new AtomicReference<>(null);
    ClusterSettings.SettingUpdater<ByteSizeValue> settingUpdater = memorySizeValueSetting.newUpdater(value::set, logger);

    assertTrue(settingUpdater.apply(Settings.builder().put("a.byte.size", "12").build(), Settings.EMPTY));
    assertEquals(new ByteSizeValue(12), value.get());

    assertTrue(settingUpdater.apply(Settings.builder().put("a.byte.size", "12b").build(), Settings.EMPTY));
    assertEquals(new ByteSizeValue(12), value.get());

    assertTrue(settingUpdater.apply(Settings.builder().put("a.byte.size", "20%").build(), Settings.EMPTY));
    assertEquals(new ByteSizeValue((int) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.2)), value.get());
}
 
Example 21
Source Project: elasticsearch-gatherer   Source File: AbstractGatherer.java    License: Apache License 2.0 5 votes vote down vote up
public Gatherer start() throws ElasticSearchException {
    //this.logger = Loggers.getLogger(getClass(), settings.globalSettings(), riverName);
    this.bulkActions = settings.getAsInt("bulk_actions", 1000);
    this.bulkSize = settings.getAsBytesSize("bulk_size", new ByteSizeValue(5, ByteSizeUnit.MB));
    this.flushInterval = settings.getAsTime("flush_interval", TimeValue.timeValueSeconds(5));
    this.concurrentRequests = settings.getAsInt("concurrent_requests", 4);
    bulkProcessor = BulkProcessor.builder(client, new BulkListener())
            .setBulkActions(bulkActions)
            .setBulkSize(bulkSize)
            .setFlushInterval(flushInterval)
            .setConcurrentRequests(concurrentRequests)
            .build();
    return this;
}
 
Example 22
Source Project: crate   Source File: RowAccountingWithEstimatorsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRowCellsAccountingCircuitBreakingWorks() throws Exception {
    RowCellsAccountingWithEstimators rowAccounting = new RowCellsAccountingWithEstimators(
        Collections.singletonList(DataTypes.INTEGER),
        ConcurrentRamAccounting.forCircuitBreaker(
            "test",
            new MemoryCircuitBreaker(
                new ByteSizeValue(10, ByteSizeUnit.BYTES), 1.01, LogManager.getLogger(RowAccountingWithEstimatorsTest.class))
        ), 0);

    expectedException.expect(CircuitBreakingException.class);
    IntStream.range(0, 3).forEach(i -> rowAccounting.accountForAndMaybeBreak(new Object[]{i}));
}
 
Example 23
Source Project: crate   Source File: MemorySizeSettingsTests.java    License: Apache License 2.0 5 votes vote down vote up
private void assertMemorySizeSetting(Setting<ByteSizeValue> setting, String settingKey, ByteSizeValue defaultValue) {
    assertThat(setting, notNullValue());
    assertThat(setting.getKey(), equalTo(settingKey));
    assertThat(setting.getProperties(), hasItem(Property.NodeScope));
    assertThat(setting.getDefault(Settings.EMPTY),
            equalTo(defaultValue));
    Settings settingWithPercentage = Settings.builder().put(settingKey, "25%").build();
    assertThat(setting.get(settingWithPercentage),
            equalTo(new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.25))));
    Settings settingWithBytesValue = Settings.builder().put(settingKey, "1024b").build();
    assertThat(setting.get(settingWithBytesValue), equalTo(new ByteSizeValue(1024)));
}
 
Example 24
Source Project: crate   Source File: ChildMemoryCircuitBreaker.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Method used to trip the breaker, delegates to the parent to determine
 * whether to trip the breaker or not
 */
private void circuitBreak(String fieldName, long bytesNeeded) {
    this.trippedCount.incrementAndGet();
    final String message = "[" + this.name + "] Data too large, data for [" + fieldName + "]" +
            " would be [" + bytesNeeded + "/" + new ByteSizeValue(bytesNeeded) + "]" +
            ", which is larger than the limit of [" +
            memoryBytesLimit + "/" + new ByteSizeValue(memoryBytesLimit) + "]";
    logger.debug("{}", message);
    throw new CircuitBreakingException(message, bytesNeeded, memoryBytesLimit);
}
 
Example 25
@Override
public void open() throws WriterException {

    bulkListener = new BulkListener();
    bulkProcessor = BulkProcessor.builder(getClient(),
            bulkListener).setBulkActions(1000).setBulkSize(
            new ByteSizeValue(5, ByteSizeUnit.MB)).setFlushInterval(
            TimeValue.timeValueSeconds(5)).setConcurrentRequests(
            1).build();
}
 
Example 26
Source Project: crate   Source File: TranslogWriter.java    License: Apache License 2.0 5 votes vote down vote up
private TranslogWriter(
    final ChannelFactory channelFactory,
    final ShardId shardId,
    final Checkpoint initialCheckpoint,
    final FileChannel channel,
    final Path path,
    final ByteSizeValue bufferSize,
    final LongSupplier globalCheckpointSupplier, LongSupplier minTranslogGenerationSupplier, TranslogHeader header,
    TragicExceptionHolder tragedy,
    final LongConsumer persistedSequenceNumberConsumer)
        throws
        IOException {
    super(initialCheckpoint.generation, channel, path, header);
    assert initialCheckpoint.offset == channel.position() :
        "initial checkpoint offset [" + initialCheckpoint.offset + "] is different than current channel position ["
            + channel.position() + "]";
    this.shardId = shardId;
    this.channelFactory = channelFactory;
    this.minTranslogGenerationSupplier = minTranslogGenerationSupplier;
    this.outputStream = new BufferedChannelOutputStream(java.nio.channels.Channels.newOutputStream(channel), bufferSize.bytesAsInt());
    this.lastSyncedCheckpoint = initialCheckpoint;
    this.totalOffset = initialCheckpoint.offset;
    assert initialCheckpoint.minSeqNo == SequenceNumbers.NO_OPS_PERFORMED : initialCheckpoint.minSeqNo;
    this.minSeqNo = initialCheckpoint.minSeqNo;
    assert initialCheckpoint.maxSeqNo == SequenceNumbers.NO_OPS_PERFORMED : initialCheckpoint.maxSeqNo;
    this.maxSeqNo = initialCheckpoint.maxSeqNo;
    assert initialCheckpoint.trimmedAboveSeqNo == SequenceNumbers.UNASSIGNED_SEQ_NO : initialCheckpoint.trimmedAboveSeqNo;
    this.globalCheckpointSupplier = globalCheckpointSupplier;
    this.nonFsyncedSequenceNumbers = new LongArrayList(64);
    this.persistedSequenceNumberConsumer = persistedSequenceNumberConsumer;
    this.seenSequenceNumbers = Assertions.ENABLED ? new HashMap<>() : null;
    this.tragedy = tragedy;
}
 
Example 27
Source Project: elasticsearch-xml   Source File: XmlXContentBuilder.java    License: Apache License 2.0 5 votes vote down vote up
public XmlXContentBuilder byteSizeField(XContentBuilderString rawFieldName, XContentBuilderString readableFieldName, ByteSizeValue byteSizeValue) throws IOException {
    if (humanReadable) {
        field(readableFieldName, byteSizeValue.toString());
    }
    field(rawFieldName, byteSizeValue.bytes());
    return this;
}
 
Example 28
Source Project: elasticsearch-xml   Source File: XmlXContentBuilder.java    License: Apache License 2.0 5 votes vote down vote up
public XmlXContentBuilder byteSizeField(XContentBuilderString rawFieldName, XContentBuilderString readableFieldName, long rawSize) throws IOException {
    if (humanReadable) {
        field(readableFieldName, new ByteSizeValue(rawSize).toString());
    }
    field(rawFieldName, rawSize);
    return this;
}
 
Example 29
Source Project: crate   Source File: HierarchyCircuitBreakerService.java    License: Apache License 2.0 5 votes vote down vote up
public static String breakingExceptionMessage(String label, long limit) {
    return String.format(
        Locale.ENGLISH,
        BREAKING_EXCEPTION_MESSAGE,
        label,
        limit,
        new ByteSizeValue(limit)
    );
}
 
Example 30
Source Project: crate   Source File: TcpTransport.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Validates the first N bytes of the message header and returns <code>false</code> if the message is
 * a ping message and has no payload ie. isn't a real user level message.
 *
 * @throws IllegalStateException    if the message is too short, less than the header or less that the header plus the message size
 * @throws HttpOnTransportException if the message has no valid header and appears to be an HTTP message
 * @throws IllegalArgumentException if the message is greater that the maximum allowed frame size. This is dependent on the available
 *                                  memory.
 */
public static boolean validateMessageHeader(BytesReference buffer) throws IOException {
    final int sizeHeaderLength = TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE;
    if (buffer.length() < sizeHeaderLength) {
        throw new IllegalStateException("message size must be >= to the header size");
    }
    int offset = 0;
    if (buffer.get(offset) != 'E' || buffer.get(offset + 1) != 'S') {
        // special handling for what is probably HTTP
        if (bufferStartsWith(buffer, offset, "GET ") ||
            bufferStartsWith(buffer, offset, "POST ") ||
            bufferStartsWith(buffer, offset, "PUT ") ||
            bufferStartsWith(buffer, offset, "HEAD ") ||
            bufferStartsWith(buffer, offset, "DELETE ") ||
            bufferStartsWith(buffer, offset, "OPTIONS ") ||
            bufferStartsWith(buffer, offset, "PATCH ") ||
            bufferStartsWith(buffer, offset, "TRACE ")) {

            throw new HttpOnTransportException("This is not an HTTP port");
        }

        // we have 6 readable bytes, show 4 (should be enough)
        throw new StreamCorruptedException("invalid internal transport message format, got ("
            + Integer.toHexString(buffer.get(offset) & 0xFF) + ","
            + Integer.toHexString(buffer.get(offset + 1) & 0xFF) + ","
            + Integer.toHexString(buffer.get(offset + 2) & 0xFF) + ","
            + Integer.toHexString(buffer.get(offset + 3) & 0xFF) + ")");
    }

    final int dataLen;
    try (StreamInput input = buffer.streamInput()) {
        input.skip(TcpHeader.MARKER_BYTES_SIZE);
        dataLen = input.readInt();
        if (dataLen == PING_DATA_SIZE) {
            // discard the messages we read and continue, this is achieved by skipping the bytes
            // and returning null
            return false;
        }
    }

    if (dataLen <= 0) {
        throw new StreamCorruptedException("invalid data length: " + dataLen);
    }
    // safety against too large frames being sent
    if (dataLen > NINETY_PER_HEAP_SIZE) {
        throw new IllegalArgumentException("transport content length received [" + new ByteSizeValue(dataLen) + "] exceeded ["
            + new ByteSizeValue(NINETY_PER_HEAP_SIZE) + "]");
    }

    if (buffer.length() < dataLen + sizeHeaderLength) {
        throw new IllegalStateException("buffer must be >= to the message size but wasn't");
    }
    return true;
}