Java Code Examples for org.rocksdb.CompactionStyle

The following examples show how to use org.rocksdb.CompactionStyle. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: RocksDBPerformanceTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 
Example 2
@Test
public void testPredefinedOptions() throws Exception {
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	// verify that we would use PredefinedOptions.DEFAULT by default.
	assertEquals(PredefinedOptions.DEFAULT, rocksDbBackend.getPredefinedOptions());

	// verify that user could configure predefined options via flink-conf.yaml
	Configuration configuration = new Configuration();
	configuration.setString(RocksDBOptions.PREDEFINED_OPTIONS, PredefinedOptions.FLASH_SSD_OPTIMIZED.name());
	rocksDbBackend = new RocksDBStateBackend(checkpointPath);
	rocksDbBackend = rocksDbBackend.configure(configuration, getClass().getClassLoader());
	assertEquals(PredefinedOptions.FLASH_SSD_OPTIMIZED, rocksDbBackend.getPredefinedOptions());

	// verify that predefined options could be set programmatically and override pre-configured one.
	rocksDbBackend.setPredefinedOptions(PredefinedOptions.SPINNING_DISK_OPTIMIZED);
	assertEquals(PredefinedOptions.SPINNING_DISK_OPTIMIZED, rocksDbBackend.getPredefinedOptions());

	try (ColumnFamilyOptions colCreated = rocksDbBackend.getColumnOptions()) {
		assertEquals(CompactionStyle.LEVEL, colCreated.compactionStyle());
	}
}
 
Example 3
Source Project: flink   Source File: RocksDBPerformanceTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 
Example 4
Source Project: flink   Source File: RocksDBStateBackendConfigTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPredefinedOptions() throws Exception {
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	// verify that we would use PredefinedOptions.DEFAULT by default.
	assertEquals(PredefinedOptions.DEFAULT, rocksDbBackend.getPredefinedOptions());

	// verify that user could configure predefined options via flink-conf.yaml
	Configuration configuration = new Configuration();
	configuration.setString(RocksDBOptions.PREDEFINED_OPTIONS, PredefinedOptions.FLASH_SSD_OPTIMIZED.name());
	rocksDbBackend = new RocksDBStateBackend(checkpointPath);
	rocksDbBackend = rocksDbBackend.configure(configuration, getClass().getClassLoader());
	assertEquals(PredefinedOptions.FLASH_SSD_OPTIMIZED, rocksDbBackend.getPredefinedOptions());

	// verify that predefined options could be set programmatically and override pre-configured one.
	rocksDbBackend.setPredefinedOptions(PredefinedOptions.SPINNING_DISK_OPTIMIZED);
	assertEquals(PredefinedOptions.SPINNING_DISK_OPTIMIZED, rocksDbBackend.getPredefinedOptions());

	try (ColumnFamilyOptions colCreated = rocksDbBackend.getColumnOptions()) {
		assertEquals(CompactionStyle.LEVEL, colCreated.compactionStyle());
	}
}
 
Example 5
Source Project: flink   Source File: RocksDBPerformanceTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 
Example 6
@Test
public void testSetConfigurableOptions() throws Exception  {
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	assertNull(rocksDbBackend.getOptions());

	DefaultConfigurableOptionsFactory customizedOptions = new DefaultConfigurableOptionsFactory()
		.setMaxBackgroundThreads(4)
		.setMaxOpenFiles(-1)
		.setCompactionStyle(CompactionStyle.LEVEL)
		.setUseDynamicLevelSize(true)
		.setTargetFileSizeBase("4MB")
		.setMaxSizeLevelBase("128 mb")
		.setWriteBufferSize("128 MB")
		.setMaxWriteBufferNumber(4)
		.setMinWriteBufferNumberToMerge(3)
		.setBlockSize("64KB")
		.setBlockCacheSize("512mb");

	rocksDbBackend.setOptions(customizedOptions);

	try (DBOptions dbOptions = rocksDbBackend.getDbOptions()) {
		assertEquals(-1, dbOptions.maxOpenFiles());
	}

	try (ColumnFamilyOptions columnOptions = rocksDbBackend.getColumnOptions()) {
		assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
		assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
		assertEquals(4 * SizeUnit.MB, columnOptions.targetFileSizeBase());
		assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
		assertEquals(4, columnOptions.maxWriteBufferNumber());
		assertEquals(3, columnOptions.minWriteBufferNumberToMerge());

		BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
		assertEquals(64 * SizeUnit.KB, tableConfig.blockSize());
		assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
	}
}
 
Example 7
Source Project: flink   Source File: RocksDBStateBackendConfigTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSetConfigurableOptions() throws Exception  {
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	assertNull(rocksDbBackend.getOptions());

	DefaultConfigurableOptionsFactory customizedOptions = new DefaultConfigurableOptionsFactory()
		.setMaxBackgroundThreads(4)
		.setMaxOpenFiles(-1)
		.setCompactionStyle(CompactionStyle.LEVEL)
		.setUseDynamicLevelSize(true)
		.setTargetFileSizeBase("4MB")
		.setMaxSizeLevelBase("128 mb")
		.setWriteBufferSize("128 MB")
		.setMaxWriteBufferNumber(4)
		.setMinWriteBufferNumberToMerge(3)
		.setBlockSize("64KB")
		.setBlockCacheSize("512mb");

	rocksDbBackend.setOptions(customizedOptions);

	try (DBOptions dbOptions = rocksDbBackend.getDbOptions()) {
		assertEquals(-1, dbOptions.maxOpenFiles());
	}

	try (ColumnFamilyOptions columnOptions = rocksDbBackend.getColumnOptions()) {
		assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
		assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
		assertEquals(4 * SizeUnit.MB, columnOptions.targetFileSizeBase());
		assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
		assertEquals(4, columnOptions.maxWriteBufferNumber());
		assertEquals(3, columnOptions.minWriteBufferNumberToMerge());

		BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
		assertEquals(64 * SizeUnit.KB, tableConfig.blockSize());
		assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
	}
}
 
Example 8
Source Project: kylin-on-parquet-v2   Source File: RocksDBLookupBuilder.java    License: Apache License 2.0 5 votes vote down vote up
public RocksDBLookupBuilder(TableDesc tableDesc, String[] keyColumns, String dbPath) {
    this.tableDesc = tableDesc;
    this.encoder = new RocksDBLookupRowEncoder(tableDesc, keyColumns);
    this.dbPath = dbPath;
    this.writeBatchSize = 500;
    this.options = new Options();
    options.setCreateIfMissing(true).setWriteBufferSize(8 * SizeUnit.KB).setMaxWriteBufferNumber(3)
            .setMaxBackgroundCompactions(5).setCompressionType(CompressionType.SNAPPY_COMPRESSION)
            .setCompactionStyle(CompactionStyle.UNIVERSAL);

}
 
Example 9
Source Project: kylin   Source File: RocksDBLookupBuilder.java    License: Apache License 2.0 5 votes vote down vote up
public RocksDBLookupBuilder(TableDesc tableDesc, String[] keyColumns, String dbPath) {
    this.tableDesc = tableDesc;
    this.encoder = new RocksDBLookupRowEncoder(tableDesc, keyColumns);
    this.dbPath = dbPath;
    this.writeBatchSize = 500;
    this.options = new Options();
    options.setCreateIfMissing(true).setWriteBufferSize(8 * SizeUnit.KB).setMaxWriteBufferNumber(3)
            .setMaxBackgroundCompactions(5).setCompressionType(CompressionType.SNAPPY_COMPRESSION)
            .setCompactionStyle(CompactionStyle.UNIVERSAL);

}
 
Example 10
Source Project: flink   Source File: RocksDBStateBackendConfigTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSetConfigurableOptions() throws Exception  {
	DefaultConfigurableOptionsFactory customizedOptions = new DefaultConfigurableOptionsFactory()
		.setMaxBackgroundThreads(4)
		.setMaxOpenFiles(-1)
		.setCompactionStyle(CompactionStyle.LEVEL)
		.setUseDynamicLevelSize(true)
		.setTargetFileSizeBase("4MB")
		.setMaxSizeLevelBase("128 mb")
		.setWriteBufferSize("128 MB")
		.setMaxWriteBufferNumber(4)
		.setMinWriteBufferNumberToMerge(3)
		.setBlockSize("64KB")
		.setBlockCacheSize("512mb");

	try (RocksDBResourceContainer optionsContainer =
			new RocksDBResourceContainer(PredefinedOptions.DEFAULT, customizedOptions)) {

		DBOptions dbOptions = optionsContainer.getDbOptions();
		assertEquals(-1, dbOptions.maxOpenFiles());

		ColumnFamilyOptions columnOptions = optionsContainer.getColumnOptions();
		assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
		assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
		assertEquals(4 * SizeUnit.MB, columnOptions.targetFileSizeBase());
		assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
		assertEquals(4, columnOptions.maxWriteBufferNumber());
		assertEquals(3, columnOptions.minWriteBufferNumberToMerge());

		BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
		assertEquals(64 * SizeUnit.KB, tableConfig.blockSize());
		assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
	}
}
 
Example 11
Source Project: journalkeeper   Source File: RocksDBConfigParser.java    License: Apache License 2.0 4 votes vote down vote up
public static Options parse(Properties properties) {
    Options options = new Options();
    options.setCompressionType(CompressionType.LZ4_COMPRESSION)
            .setCompactionStyle(CompactionStyle.LEVEL);

    BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
    options.setTableFormatConfig(tableOptions);

    for (String key : properties.stringPropertyNames()) {
        String prefix = null;
        Object configInstance = null;

        if (key.startsWith(RocksDBConfigs.OPTIONS_PREFIX)) {
            prefix = RocksDBConfigs.OPTIONS_PREFIX;
            configInstance = options;
        } else if (key.startsWith(RocksDBConfigs.TABLE_OPTIONS_PREFIX)) {
            prefix = RocksDBConfigs.TABLE_OPTIONS_PREFIX;
            configInstance = tableOptions;
        } else {
            continue;
        }

        String fieldKey = key.substring(prefix.length(), key.length());
        String value = properties.getProperty(key);

        try {
            Method setterMethod = findSetterMethod(configInstance.getClass(), fieldKey);
            if (setterMethod == null) {
                logger.warn("parse config error, method not found, key: {}, value: {}", key, value);
                continue;
            }
            setterMethod.invoke(configInstance, PropertyUtils.convert(value, setterMethod.getParameters()[0].getType()));
        } catch (Exception e) {
            logger.error("parse config error, key: {}, value: {}", key, value, e);
        }
    }

    if (properties.containsKey(RocksDBConfigs.FILTER_BITSPER_KEY)) {
        tableOptions.setFilterPolicy(new BloomFilter(
                PropertyUtils.convertInt(properties.getProperty(RocksDBConfigs.FILTER_BITSPER_KEY), 0)));
    }

    return options;
}
 
Example 12
private CompactionStyle getCompactionStyle() {
	return CompactionStyle.valueOf(getInternal(COMPACTION_STYLE.key()).toUpperCase());
}
 
Example 13
public DefaultConfigurableOptionsFactory setCompactionStyle(CompactionStyle compactionStyle) {
	setInternal(COMPACTION_STYLE.key(), compactionStyle.name());
	return this;
}
 
Example 14
@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 
Example 15
@Test
public void testConfigurableOptionsFromConfig() throws IOException {
	Configuration configuration = new Configuration();
	DefaultConfigurableOptionsFactory defaultOptionsFactory = new DefaultConfigurableOptionsFactory();
	assertTrue(defaultOptionsFactory.configure(configuration).getConfiguredOptions().isEmpty());

	// verify illegal configuration
	{
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_BACKGROUND_THREADS, "-1");
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_WRITE_BUFFER_NUMBER, "-1");
		verifyIllegalArgument(RocksDBConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE, "-1");

		verifyIllegalArgument(RocksDBConfigurableOptions.TARGET_FILE_SIZE_BASE, "0KB");
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_SIZE_LEVEL_BASE, "1BB");
		verifyIllegalArgument(RocksDBConfigurableOptions.WRITE_BUFFER_SIZE, "-1KB");
		verifyIllegalArgument(RocksDBConfigurableOptions.BLOCK_SIZE, "0MB");
		verifyIllegalArgument(RocksDBConfigurableOptions.BLOCK_CACHE_SIZE, "0");

		verifyIllegalArgument(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE, "1");

		verifyIllegalArgument(RocksDBConfigurableOptions.COMPACTION_STYLE, "LEV");
	}

	// verify legal configuration
	{
		configuration.setString(RocksDBConfigurableOptions.COMPACTION_STYLE, "level");
		configuration.setString(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE, "TRUE");
		configuration.setString(RocksDBConfigurableOptions.TARGET_FILE_SIZE_BASE, "8 mb");
		configuration.setString(RocksDBConfigurableOptions.MAX_SIZE_LEVEL_BASE, "128MB");
		configuration.setString(RocksDBConfigurableOptions.MAX_BACKGROUND_THREADS, "4");
		configuration.setString(RocksDBConfigurableOptions.MAX_WRITE_BUFFER_NUMBER, "4");
		configuration.setString(RocksDBConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE, "2");
		configuration.setString(RocksDBConfigurableOptions.WRITE_BUFFER_SIZE, "64 MB");
		configuration.setString(RocksDBConfigurableOptions.BLOCK_SIZE, "4 kb");
		configuration.setString(RocksDBConfigurableOptions.BLOCK_CACHE_SIZE, "512 mb");

		DefaultConfigurableOptionsFactory optionsFactory = new DefaultConfigurableOptionsFactory();
		optionsFactory.configure(configuration);
		String checkpointPath = tempFolder.newFolder().toURI().toString();
		RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);
		rocksDbBackend.setOptions(optionsFactory);

		try (DBOptions dbOptions = rocksDbBackend.getDbOptions()) {
			assertEquals(-1, dbOptions.maxOpenFiles());
		}

		try (ColumnFamilyOptions columnOptions = rocksDbBackend.getColumnOptions()) {
			assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
			assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
			assertEquals(8 * SizeUnit.MB, columnOptions.targetFileSizeBase());
			assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
			assertEquals(4, columnOptions.maxWriteBufferNumber());
			assertEquals(2, columnOptions.minWriteBufferNumberToMerge());
			assertEquals(64 * SizeUnit.MB, columnOptions.writeBufferSize());

			BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
			assertEquals(4 * SizeUnit.KB, tableConfig.blockSize());
			assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
		}
	}
}
 
Example 16
@Override
public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions) {
	return currentOptions.setCompactionStyle(CompactionStyle.UNIVERSAL);
}
 
Example 17
Source Project: sofa-jraft   Source File: StorageOptionsFactory.java    License: Apache License 2.0 4 votes vote down vote up
public static ColumnFamilyOptions getDefaultRocksDBColumnFamilyOptions() {
    final ColumnFamilyOptions opts = new ColumnFamilyOptions();

    // Flushing options:
    // write_buffer_size sets the size of a single mem_table. Once mem_table exceeds
    // this size, it is marked immutable and a new one is created.
    opts.setWriteBufferSize(64 * SizeUnit.MB);

    // Flushing options:
    // max_write_buffer_number sets the maximum number of mem_tables, both active
    // and immutable.  If the active mem_table fills up and the total number of
    // mem_tables is larger than max_write_buffer_number we stall further writes.
    // This may happen if the flush process is slower than the write rate.
    opts.setMaxWriteBufferNumber(3);

    // Flushing options:
    // min_write_buffer_number_to_merge is the minimum number of mem_tables to be
    // merged before flushing to storage. For example, if this option is set to 2,
    // immutable mem_tables are only flushed when there are two of them - a single
    // immutable mem_table will never be flushed.  If multiple mem_tables are merged
    // together, less data may be written to storage since two updates are merged to
    // a single key. However, every Get() must traverse all immutable mem_tables
    // linearly to check if the key is there. Setting this option too high may hurt
    // read performance.
    opts.setMinWriteBufferNumberToMerge(1);

    // Level Style Compaction:
    // level0_file_num_compaction_trigger -- Once level 0 reaches this number of
    // files, L0->L1 compaction is triggered. We can therefore estimate level 0
    // size in stable state as
    // write_buffer_size * min_write_buffer_number_to_merge * level0_file_num_compaction_trigger.
    opts.setLevel0FileNumCompactionTrigger(10);

    // Soft limit on number of level-0 files. We start slowing down writes at this
    // point. A value 0 means that no writing slow down will be triggered by number
    // of files in level-0.
    opts.setLevel0SlowdownWritesTrigger(20);

    // Maximum number of level-0 files.  We stop writes at this point.
    opts.setLevel0StopWritesTrigger(40);

    // Level Style Compaction:
    // max_bytes_for_level_base and max_bytes_for_level_multiplier
    //  -- max_bytes_for_level_base is total size of level 1. As mentioned, we
    // recommend that this be around the size of level 0. Each subsequent level
    // is max_bytes_for_level_multiplier larger than previous one. The default
    // is 10 and we do not recommend changing that.
    opts.setMaxBytesForLevelBase(512 * SizeUnit.MB);

    // Level Style Compaction:
    // target_file_size_base and target_file_size_multiplier
    //  -- Files in level 1 will have target_file_size_base bytes. Each next
    // level's file size will be target_file_size_multiplier bigger than previous
    // one. However, by default target_file_size_multiplier is 1, so files in all
    // L1..LMax levels are equal. Increasing target_file_size_base will reduce total
    // number of database files, which is generally a good thing. We recommend setting
    // target_file_size_base to be max_bytes_for_level_base / 10, so that there are
    // 10 files in level 1.
    opts.setTargetFileSizeBase(64 * SizeUnit.MB);

    // If prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0,
    // create prefix bloom for memtable with the size of
    // write_buffer_size * memtable_prefix_bloom_size_ratio.
    // If it is larger than 0.25, it is santinized to 0.25.
    opts.setMemtablePrefixBloomSizeRatio(0.125);

    // Seems like the rocksDB jni for Windows doesn't come linked with any of the
    // compression type
    if (!Platform.isWindows()) {
        opts.setCompressionType(CompressionType.LZ4_COMPRESSION) //
            .setCompactionStyle(CompactionStyle.LEVEL) //
            .optimizeLevelStyleCompaction();
    }

    // https://github.com/facebook/rocksdb/pull/5744
    opts.setForceConsistencyChecks(true);

    return opts;
}
 
Example 18
Source Project: flink   Source File: DefaultConfigurableOptionsFactory.java    License: Apache License 2.0 4 votes vote down vote up
private CompactionStyle getCompactionStyle() {
	return CompactionStyle.valueOf(getInternal(COMPACTION_STYLE.key()).toUpperCase());
}
 
Example 19
Source Project: flink   Source File: DefaultConfigurableOptionsFactory.java    License: Apache License 2.0 4 votes vote down vote up
public DefaultConfigurableOptionsFactory setCompactionStyle(CompactionStyle compactionStyle) {
	setInternal(COMPACTION_STYLE.key(), compactionStyle.name());
	return this;
}
 
Example 20
Source Project: flink   Source File: RocksDBListStatePerformanceTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 
Example 21
Source Project: flink   Source File: RocksDBStateBackendConfigTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testConfigurableOptionsFromConfig() throws IOException {
	Configuration configuration = new Configuration();
	DefaultConfigurableOptionsFactory defaultOptionsFactory = new DefaultConfigurableOptionsFactory();
	assertTrue(defaultOptionsFactory.configure(configuration).getConfiguredOptions().isEmpty());

	// verify illegal configuration
	{
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_BACKGROUND_THREADS, "-1");
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_WRITE_BUFFER_NUMBER, "-1");
		verifyIllegalArgument(RocksDBConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE, "-1");

		verifyIllegalArgument(RocksDBConfigurableOptions.TARGET_FILE_SIZE_BASE, "0KB");
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_SIZE_LEVEL_BASE, "1BB");
		verifyIllegalArgument(RocksDBConfigurableOptions.WRITE_BUFFER_SIZE, "-1KB");
		verifyIllegalArgument(RocksDBConfigurableOptions.BLOCK_SIZE, "0MB");
		verifyIllegalArgument(RocksDBConfigurableOptions.BLOCK_CACHE_SIZE, "0");

		verifyIllegalArgument(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE, "1");

		verifyIllegalArgument(RocksDBConfigurableOptions.COMPACTION_STYLE, "LEV");
	}

	// verify legal configuration
	{
		configuration.setString(RocksDBConfigurableOptions.COMPACTION_STYLE, "level");
		configuration.setString(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE, "TRUE");
		configuration.setString(RocksDBConfigurableOptions.TARGET_FILE_SIZE_BASE, "8 mb");
		configuration.setString(RocksDBConfigurableOptions.MAX_SIZE_LEVEL_BASE, "128MB");
		configuration.setString(RocksDBConfigurableOptions.MAX_BACKGROUND_THREADS, "4");
		configuration.setString(RocksDBConfigurableOptions.MAX_WRITE_BUFFER_NUMBER, "4");
		configuration.setString(RocksDBConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE, "2");
		configuration.setString(RocksDBConfigurableOptions.WRITE_BUFFER_SIZE, "64 MB");
		configuration.setString(RocksDBConfigurableOptions.BLOCK_SIZE, "4 kb");
		configuration.setString(RocksDBConfigurableOptions.BLOCK_CACHE_SIZE, "512 mb");

		DefaultConfigurableOptionsFactory optionsFactory = new DefaultConfigurableOptionsFactory();
		optionsFactory.configure(configuration);
		String checkpointPath = tempFolder.newFolder().toURI().toString();
		RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);
		rocksDbBackend.setOptions(optionsFactory);

		try (DBOptions dbOptions = rocksDbBackend.getDbOptions()) {
			assertEquals(-1, dbOptions.maxOpenFiles());
		}

		try (ColumnFamilyOptions columnOptions = rocksDbBackend.getColumnOptions()) {
			assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
			assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
			assertEquals(8 * SizeUnit.MB, columnOptions.targetFileSizeBase());
			assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
			assertEquals(4, columnOptions.maxWriteBufferNumber());
			assertEquals(2, columnOptions.minWriteBufferNumberToMerge());
			assertEquals(64 * SizeUnit.MB, columnOptions.writeBufferSize());

			BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
			assertEquals(4 * SizeUnit.KB, tableConfig.blockSize());
			assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
		}
	}
}
 
Example 22
Source Project: flink   Source File: RocksDBStateBackendConfigTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions) {
	return currentOptions.setCompactionStyle(CompactionStyle.UNIVERSAL);
}
 
Example 23
@Override
public CompactionStyle compactionStyle() {
    return columnFamilyOptions.compactionStyle();
}
 
Example 24
@Override
public Options setCompactionStyle(final CompactionStyle compactionStyle) {
    columnFamilyOptions.setCompactionStyle(compactionStyle);
    return this;
}
 
Example 25
Source Project: flink   Source File: DefaultConfigurableOptionsFactory.java    License: Apache License 2.0 4 votes vote down vote up
private CompactionStyle getCompactionStyle() {
	return CompactionStyle.valueOf(getInternal(COMPACTION_STYLE.key()).toUpperCase());
}
 
Example 26
Source Project: flink   Source File: DefaultConfigurableOptionsFactory.java    License: Apache License 2.0 4 votes vote down vote up
public DefaultConfigurableOptionsFactory setCompactionStyle(CompactionStyle compactionStyle) {
	setInternal(COMPACTION_STYLE.key(), compactionStyle.name());
	return this;
}
 
Example 27
Source Project: flink   Source File: RocksDBListStatePerformanceTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 
Example 28
Source Project: flink   Source File: RocksDBStateBackendConfigTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testConfigurableOptionsFromConfig() throws Exception {
	Configuration configuration = new Configuration();
	DefaultConfigurableOptionsFactory defaultOptionsFactory = new DefaultConfigurableOptionsFactory();
	assertTrue(defaultOptionsFactory.configure(configuration).getConfiguredOptions().isEmpty());

	// verify illegal configuration
	{
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_BACKGROUND_THREADS, "-1");
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_WRITE_BUFFER_NUMBER, "-1");
		verifyIllegalArgument(RocksDBConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE, "-1");

		verifyIllegalArgument(RocksDBConfigurableOptions.TARGET_FILE_SIZE_BASE, "0KB");
		verifyIllegalArgument(RocksDBConfigurableOptions.MAX_SIZE_LEVEL_BASE, "1BB");
		verifyIllegalArgument(RocksDBConfigurableOptions.WRITE_BUFFER_SIZE, "-1KB");
		verifyIllegalArgument(RocksDBConfigurableOptions.BLOCK_SIZE, "0MB");
		verifyIllegalArgument(RocksDBConfigurableOptions.BLOCK_CACHE_SIZE, "0");

		verifyIllegalArgument(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE, "1");

		verifyIllegalArgument(RocksDBConfigurableOptions.COMPACTION_STYLE, "LEV");
	}

	// verify legal configuration
	{
		configuration.setString(RocksDBConfigurableOptions.COMPACTION_STYLE.key(), "level");
		configuration.setString(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE.key(), "TRUE");
		configuration.setString(RocksDBConfigurableOptions.TARGET_FILE_SIZE_BASE.key(), "8 mb");
		configuration.setString(RocksDBConfigurableOptions.MAX_SIZE_LEVEL_BASE.key(), "128MB");
		configuration.setString(RocksDBConfigurableOptions.MAX_BACKGROUND_THREADS.key(), "4");
		configuration.setString(RocksDBConfigurableOptions.MAX_WRITE_BUFFER_NUMBER.key(), "4");
		configuration.setString(RocksDBConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE.key(), "2");
		configuration.setString(RocksDBConfigurableOptions.WRITE_BUFFER_SIZE.key(), "64 MB");
		configuration.setString(RocksDBConfigurableOptions.BLOCK_SIZE.key(), "4 kb");
		configuration.setString(RocksDBConfigurableOptions.BLOCK_CACHE_SIZE.key(), "512 mb");

		DefaultConfigurableOptionsFactory optionsFactory = new DefaultConfigurableOptionsFactory();
		optionsFactory.configure(configuration);

		try (RocksDBResourceContainer optionsContainer =
				new RocksDBResourceContainer(PredefinedOptions.DEFAULT, optionsFactory)) {

			DBOptions dbOptions = optionsContainer.getDbOptions();
			assertEquals(-1, dbOptions.maxOpenFiles());

			ColumnFamilyOptions columnOptions = optionsContainer.getColumnOptions();
			assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
			assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
			assertEquals(8 * SizeUnit.MB, columnOptions.targetFileSizeBase());
			assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
			assertEquals(4, columnOptions.maxWriteBufferNumber());
			assertEquals(2, columnOptions.minWriteBufferNumberToMerge());
			assertEquals(64 * SizeUnit.MB, columnOptions.writeBufferSize());

			BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
			assertEquals(4 * SizeUnit.KB, tableConfig.blockSize());
			assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
		}
	}
}
 
Example 29
Source Project: flink   Source File: RocksDBStateBackendConfigTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
	return currentOptions.setCompactionStyle(CompactionStyle.UNIVERSAL);
}
 
Example 30
Source Project: jstorm   Source File: RocksDbUnitTest.java    License: Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) {
    Map conf = JStormHelper.LoadConf(args[0]);
    putNum = JStormUtils.parseInt(conf.get("put.number"), 100);
    isFlush = JStormUtils.parseBoolean(conf.get("is.flush"), true);
    isCheckpoint = JStormUtils.parseBoolean(conf.get("is.checkpoint"), true);
    sleepTime = JStormUtils.parseInt(conf.get("sleep.time"), 5000);
    compactionInterval = JStormUtils.parseInt(conf.get("compaction.interval"), 30000);
    flushInterval = JStormUtils.parseInt(conf.get("flush.interval"), 3000);
    isCompaction = JStormUtils.parseBoolean(conf.get("is.compaction"), true);
    fileSizeBase = JStormUtils.parseLong(conf.get("file.size.base"), 10 * SizeUnit.KB);
    levelNum = JStormUtils.parseInt(conf.get("db.level.num"), 1);
    compactionTriggerNum = JStormUtils.parseInt(conf.get("db.compaction.trigger.num"), 4);
    LOG.info("Conf={}", conf);
    
    RocksDB db;
    File file = new File(cpPath);
    file.mkdirs();

    List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
    try {
        Options options = new Options();
        options.setCreateMissingColumnFamilies(true);
        options.setCreateIfMissing(true);
        options.setTargetFileSizeBase(fileSizeBase);
        options.setMaxBackgroundFlushes(2);
        options.setMaxBackgroundCompactions(2);
        options.setCompactionStyle(CompactionStyle.LEVEL);
        options.setNumLevels(levelNum);
        options.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);

        DBOptions dbOptions = new DBOptions();
        dbOptions.setCreateMissingColumnFamilies(true);
        dbOptions.setCreateIfMissing(true);
        dbOptions.setMaxBackgroundFlushes(2);
        dbOptions.setMaxBackgroundCompactions(2);
        ColumnFamilyOptions familyOptions = new ColumnFamilyOptions();
        familyOptions.setTargetFileSizeBase(fileSizeBase);
        familyOptions.setCompactionStyle(CompactionStyle.LEVEL);
        familyOptions.setNumLevels(levelNum);
        familyOptions.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);
        List<byte[]> families = RocksDB.listColumnFamilies(options, dbPath);
        List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
        if (families != null) {
            for (byte[] bytes : families) {
                columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, familyOptions));
                LOG.info("Load colum family of {}", new String(bytes));
            }
        }
        
        if (columnFamilyDescriptors.size() > 0) {
            db = RocksDB.open(dbOptions, dbPath, columnFamilyDescriptors, columnFamilyHandles);
        } else {
            db = RocksDB.open(options, dbPath);
        }
    } catch (RocksDBException e) {
        LOG.error("Failed to open db", e);
        return;
    }

    rocksDbTest(db, columnFamilyHandles);
    
    db.close();
}