Java Code Examples for org.rocksdb.DBOptions

The following examples show how to use org.rocksdb.DBOptions. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop-ozone   Source File: DBStoreBuilder.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Builds a DBStore instance and returns that.
 *
 * @return DBStore
 */
public DBStore build() throws IOException {
  if(StringUtil.isBlank(dbname) || (dbPath == null)) {
    LOG.error("Required Parameter missing.");
    throw new IOException("Required parameter is missing. Please make sure "
        + "sure Path and DB name is provided.");
  }
  processDBProfile();
  processTables();
  DBOptions options = getDbProfile();

  WriteOptions writeOptions = new WriteOptions();
  writeOptions.setSync(rocksDBConfiguration.getSyncOption());


  File dbFile = getDBFile();
  if (!dbFile.getParentFile().exists()) {
    throw new IOException("The DB destination directory should exist.");
  }
  return new RDBStore(dbFile, options, writeOptions, tables, registry);
}
 
Example 2
Source Project: hadoop-ozone   Source File: TestTypedRDBTableStore.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);

  Set<TableConfig> configSet = new HashSet<>();
  for (String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);

  codecRegistry = new CodecRegistry();

}
 
Example 3
Source Project: hadoop-ozone   Source File: TestRDBStore.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);
  configSet = new HashSet<>();
  for(String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);
}
 
Example 4
Source Project: hadoop-ozone   Source File: TestDBConfigFromFile.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void readFromFile() throws IOException {
  final List<String> families =
      Arrays.asList(StringUtils.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY),
          "First", "Second", "Third",
          "Fourth", "Fifth",
          "Sixth");
  final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
      new ArrayList<>();
  for (String family : families) {
    columnFamilyDescriptors.add(
        new ColumnFamilyDescriptor(family.getBytes(StandardCharsets.UTF_8),
            new ColumnFamilyOptions()));
  }

  final DBOptions options = DBConfigFromFile.readFromFile(DB_FILE,
      columnFamilyDescriptors);

  // Some Random Values Defined in the test.db.ini, we verify that we are
  // able to get values that are defined in the test.db.ini.
  Assert.assertNotNull(options);
  Assert.assertEquals(551615L, options.maxManifestFileSize());
  Assert.assertEquals(1000L, options.keepLogFileNum());
  Assert.assertEquals(1048576, options.writableFileMaxBufferSize());
}
 
Example 5
Source Project: hadoop-ozone   Source File: TestDBConfigFromFile.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void readFromFileInvalidConfig() throws IOException {
  final List<String> families =
      Arrays.asList(StringUtils.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY),
          "First", "Second", "Third",
          "Fourth", "Fifth",
          "Sixth");
  final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
      new ArrayList<>();
  for (String family : families) {
    columnFamilyDescriptors.add(
        new ColumnFamilyDescriptor(family.getBytes(StandardCharsets.UTF_8),
            new ColumnFamilyOptions()));
  }

  final DBOptions options = DBConfigFromFile.readFromFile("badfile.db.ini",
      columnFamilyDescriptors);

  // This has to return a Null, since we have config defined for badfile.db
  Assert.assertNull(options);
}
 
Example 6
Source Project: hadoop-ozone   Source File: TestRDBTableStore.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);

  Set<TableConfig> configSet = new HashSet<>();
  for(String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);
}
 
Example 7
private static org.rocksdb.Logger createRocksDbNativeLogger() {
	if (LOG.isDebugEnabled()) {
		// options are always needed for org.rocksdb.Logger construction (no other constructor)
		// the logger level gets configured from the options in native code
		try (DBOptions opts = new DBOptions().setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL)) {
			return new org.rocksdb.Logger(opts) {
				@Override
				protected void log(InfoLogLevel infoLogLevel, String logMsg) {
					LOG.debug("RocksDB filter native code log: " + logMsg);
				}
			};
		}
	} else {
		return null;
	}
}
 
Example 8
Source Project: flink   Source File: RocksDBResourceContainerTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Guard that {@link RocksDBResourceContainer#getDbOptions()} shares the same {@link WriteBufferManager} instance
 * if the {@link RocksDBResourceContainer} instance is initiated with {@link OpaqueMemoryResource}.
 *
 * @throws Exception if unexpected error happened.
 */
@Test
public void testGetDbOptionsWithSharedResources() throws Exception {
	final int optionNumber = 20;
	OpaqueMemoryResource<RocksDBSharedResources> sharedResources = getSharedResources();
	RocksDBResourceContainer container =
		new RocksDBResourceContainer(PredefinedOptions.DEFAULT, null, sharedResources);
	HashSet<WriteBufferManager> writeBufferManagers = new HashSet<>();
	for (int i = 0; i < optionNumber; i++) {
		DBOptions dbOptions = container.getDbOptions();
		WriteBufferManager writeBufferManager = getWriteBufferManager(dbOptions);
		writeBufferManagers.add(writeBufferManager);
	}
	assertThat(writeBufferManagers.size(), is(1));
	assertThat(writeBufferManagers.iterator().next(),
		is(sharedResources.getResourceHandle().getWriteBufferManager()));
	container.close();
}
 
Example 9
Source Project: flink   Source File: RocksDbTtlCompactFiltersManager.java    License: Apache License 2.0 6 votes vote down vote up
private static org.rocksdb.Logger createRocksDbNativeLogger() {
	if (LOG.isDebugEnabled()) {
		// options are always needed for org.rocksdb.Logger construction (no other constructor)
		// the logger level gets configured from the options in native code
		try (DBOptions opts = new DBOptions().setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL)) {
			return new org.rocksdb.Logger(opts) {
				@Override
				protected void log(InfoLogLevel infoLogLevel, String logMsg) {
					LOG.debug("RocksDB filter native code log: " + logMsg);
				}
			};
		}
	} else {
		return null;
	}
}
 
Example 10
Source Project: flink   Source File: RocksDBOperationsUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPathExceptionOnWindows() throws Exception {
	assumeTrue(OperatingSystem.isWindows());

	final File folder = TMP_DIR.newFolder();
	final File rocksDir = new File(folder, getLongString(247 - folder.getAbsolutePath().length()));

	Files.createDirectories(rocksDir.toPath());

	try (DBOptions dbOptions = new DBOptions().setCreateIfMissing(true);
		ColumnFamilyOptions colOptions = new ColumnFamilyOptions()) {

		RocksDB rocks = RocksDBOperationUtils.openDB(
				rocksDir.getAbsolutePath(),
				Collections.emptyList(),
				Collections.emptyList(),
				colOptions, dbOptions);
		rocks.close();

		// do not provoke a test failure if this passes, because some setups may actually
		// support long paths, in which case: great!
	}
	catch (IOException e) {
		assertThat(e.getMessage(), containsString("longer than the directory path length limit for Windows"));
	}
}
 
Example 11
Source Project: fasten   Source File: KnowledgeBase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("resource")
public static KnowledgeBase getInstance(final String kbDir, final String kbMetadataPathname, final boolean readOnly) throws RocksDBException, ClassNotFoundException, IOException {
	final boolean metadataExists = new File(kbMetadataPathname).exists();
	final boolean kbDirExists = new File(kbDir).exists();
	if (metadataExists != kbDirExists) throw new IllegalArgumentException("Either both or none of the knowledge-base directory and metadata must exist");

	RocksDB.loadLibrary();
	final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions().setCompressionType(CompressionType.LZ4_COMPRESSION);
	final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true);
	final List<ColumnFamilyDescriptor> cfDescriptors = Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions), new ColumnFamilyDescriptor(GID2URI, cfOptions), new ColumnFamilyDescriptor(URI2GID, cfOptions));

	final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
	final RocksDB db = readOnly ? RocksDB.openReadOnly(dbOptions, kbDir, cfDescriptors, columnFamilyHandles) : RocksDB.open(dbOptions, kbDir, cfDescriptors, columnFamilyHandles);

	final KnowledgeBase kb;
	if (metadataExists) {
		kb = (KnowledgeBase) BinIO.loadObject(kbMetadataPathname);
		kb.readOnly = readOnly;
		kb.callGraphDB = db;
		kb.defaultHandle = columnFamilyHandles.get(0);
		kb.gid2uriFamilyHandle = columnFamilyHandles.get(1);
		kb.uri2gidFamilyHandle = columnFamilyHandles.get(2);
	} else kb = new KnowledgeBase(db, columnFamilyHandles.get(0), columnFamilyHandles.get(1), columnFamilyHandles.get(2), kbMetadataPathname, readOnly);
	return kb;
}
 
Example 12
Source Project: dremio-oss   Source File: ByteStoreManager.java    License: Apache License 2.0 6 votes vote down vote up
private void registerMetrics(DBOptions dbOptions) {
  // calling DBOptions.statisticsPtr() will create a Statistics object that will collect various stats from RocksDB and
  // will introduce a 5-10% overhead
  if(!COLLECT_METRICS) {
    return;
  }

  final Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  dbOptions.setStatistics(statistics);
  // for now, let's add all ticker stats as gauge metrics
  for (TickerType tickerType : TickerType.values()) {
    if (tickerType == TickerType.TICKER_ENUM_MAX) {
      continue;
    }

    Metrics.newGauge(Metrics.join(METRICS_PREFIX, tickerType.name()), () -> statistics.getTickerCount(tickerType));
  }
  // Note that Statistics also contains various histogram metrics, but those cannot be easily tracked through our metrics
}
 
Example 13
Source Project: act   Source File: SQLConnection.java    License: GNU General Public License v3.0 6 votes vote down vote up
public Pair<RocksDB, Map<String, ColumnFamilyHandle>> openSupportingIndex(File supportingIndex)
    throws RocksDBException {
  List<FromBrendaDB> instances = BrendaSupportingEntries.allFromBrendaDBInstances();
  List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(instances.size() + 1);
  columnFamilyDescriptors.add(new ColumnFamilyDescriptor("default".getBytes()));
  for (FromBrendaDB instance : instances) {
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(instance.getColumnFamilyName().getBytes()));
  }
  List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size());

  DBOptions dbOptions = new DBOptions();
  dbOptions.setCreateIfMissing(false);
  RocksDB rocksDB = RocksDB.open(dbOptions, supportingIndex.getAbsolutePath(),
      columnFamilyDescriptors, columnFamilyHandles);
  Map<String, ColumnFamilyHandle> columnFamilyHandleMap = new HashMap<>(columnFamilyHandles.size());
  // TODO: can we zip these together more easily w/ Java 8?

  for (int i = 0; i < columnFamilyDescriptors.size(); i++) {
    ColumnFamilyDescriptor cfd = columnFamilyDescriptors.get(i);
    ColumnFamilyHandle cfh = columnFamilyHandles.get(i);
    columnFamilyHandleMap.put(new String(cfd.columnFamilyName(), BrendaSupportingEntries.UTF8), cfh);
  }

  return Pair.of(rocksDB, columnFamilyHandleMap);
}
 
Example 14
Source Project: hudi   Source File: RocksDBDAO.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Helper to load managed column family descriptors.
 */
private List<ColumnFamilyDescriptor> loadManagedColumnFamilies(DBOptions dbOptions) throws RocksDBException {
  final List<ColumnFamilyDescriptor> managedColumnFamilies = new ArrayList<>();
  final Options options = new Options(dbOptions, new ColumnFamilyOptions());
  List<byte[]> existing = RocksDB.listColumnFamilies(options, rocksDBBasePath);

  if (existing.isEmpty()) {
    LOG.info("No column family found. Loading default");
    managedColumnFamilies.add(getColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
  } else {
    LOG.info("Loading column families :" + existing.stream().map(String::new).collect(Collectors.toList()));
    managedColumnFamilies
        .addAll(existing.stream().map(RocksDBDAO::getColumnFamilyDescriptor).collect(Collectors.toList()));
  }
  return managedColumnFamilies;
}
 
Example 15
Source Project: flink   Source File: RocksDBResourceContainer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Gets the RocksDB {@link DBOptions} to be used for RocksDB instances.
 */
public DBOptions getDbOptions() {
	// initial options from pre-defined profile
	DBOptions opt = predefinedOptions.createDBOptions(handlesToClose);
	handlesToClose.add(opt);

	// add user-defined options factory, if specified
	if (optionsFactory != null) {
		opt = optionsFactory.createDBOptions(opt, handlesToClose);
	}

	// add necessary default options
	opt = opt.setCreateIfMissing(true);

	// if sharedResources is non-null, use the write buffer manager from it.
	if (sharedResources != null) {
		opt.setWriteBufferManager(sharedResources.getResourceHandle().getWriteBufferManager());
	}

	return opt;
}
 
Example 16
Source Project: flink   Source File: RocksDBStateBackendConfigTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPredefinedOptionsEnum() {
	ArrayList<AutoCloseable> handlesToClose = new ArrayList<>();
	for (PredefinedOptions o : PredefinedOptions.values()) {
		try (DBOptions opt = o.createDBOptions(handlesToClose)) {
			assertNotNull(opt);
		}
	}
	handlesToClose.forEach(IOUtils::closeQuietly);
	handlesToClose.clear();
}
 
Example 17
Source Project: Flink-CEPplus   Source File: RocksDBOperationUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static RocksDB openDB(
	String path,
	List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors,
	List<ColumnFamilyHandle> stateColumnFamilyHandles,
	ColumnFamilyOptions columnFamilyOptions,
	DBOptions dbOptions) throws IOException {
	List<ColumnFamilyDescriptor> columnFamilyDescriptors =
		new ArrayList<>(1 + stateColumnFamilyDescriptors.size());

	// we add the required descriptor for the default CF in FIRST position, see
	// https://github.com/facebook/rocksdb/wiki/RocksJava-Basics#opening-a-database-with-column-families
	columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions));
	columnFamilyDescriptors.addAll(stateColumnFamilyDescriptors);

	RocksDB dbRef;

	try {
		dbRef = RocksDB.open(
			Preconditions.checkNotNull(dbOptions),
			Preconditions.checkNotNull(path),
			columnFamilyDescriptors,
			stateColumnFamilyHandles);
	} catch (RocksDBException e) {
		IOUtils.closeQuietly(columnFamilyOptions);
		columnFamilyDescriptors.forEach((cfd) -> IOUtils.closeQuietly(cfd.getOptions()));
		throw new IOException("Error while opening RocksDB instance.", e);
	}

	// requested + default CF
	Preconditions.checkState(1 + stateColumnFamilyDescriptors.size() == stateColumnFamilyHandles.size(),
		"Not all requested column family handles have been created");
	return dbRef;
}
 
Example 18
protected AbstractRocksDBRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfTransferringThreads = numberOfTransferringThreads;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.userCodeClassLoader = userCodeClassLoader;
	this.kvStateInformation = kvStateInformation;
	this.keySerializerProvider = keySerializerProvider;
	this.instanceBasePath = instanceBasePath;
	this.instanceRocksDBPath = instanceRocksDBPath;
	this.dbPath = instanceRocksDBPath.getAbsolutePath();
	this.dbOptions = dbOptions;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.nativeMetricOptions = nativeMetricOptions;
	this.metricGroup = metricGroup;
	this.restoreStateHandles = stateHandles;
	this.ttlCompactFiltersManager = ttlCompactFiltersManager;
	this.columnFamilyHandles = new ArrayList<>(1);
	this.columnFamilyDescriptors = Collections.emptyList();
}
 
Example 19
Source Project: Flink-CEPplus   Source File: RocksDBFullRestoreOperation.java    License: Apache License 2.0 5 votes vote down vote up
public RocksDBFullRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	super(
		keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
}
 
Example 20
public RocksDBIncrementalRestoreOperation(
	String operatorIdentifier,
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
	this.operatorIdentifier = operatorIdentifier;
	this.restoredSstFiles = new TreeMap<>();
	this.lastCompletedCheckpointId = -1L;
	this.backendUID = UUID.randomUUID();
}
 
Example 21
Source Project: Flink-CEPplus   Source File: RocksDBNoneRestoreOperation.java    License: Apache License 2.0 5 votes vote down vote up
public RocksDBNoneRestoreOperation(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfTransferringThreads,
	CloseableRegistry cancelStreamRegistry,
	ClassLoader userCodeClassLoader,
	Map<String, RocksDbKvStateInfo> kvStateInformation,
	StateSerializerProvider<K> keySerializerProvider,
	File instanceBasePath,
	File instanceRocksDBPath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	RocksDBNativeMetricOptions nativeMetricOptions,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> restoreStateHandles,
	@Nonnull RocksDbTtlCompactFiltersManager ttlCompactFiltersManager
) {
	super(keyGroupRange,
		keyGroupPrefixBytes,
		numberOfTransferringThreads,
		cancelStreamRegistry,
		userCodeClassLoader,
		kvStateInformation,
		keySerializerProvider,
		instanceBasePath,
		instanceRocksDBPath,
		dbOptions,
		columnFamilyOptionsFactory,
		nativeMetricOptions,
		metricGroup,
		restoreStateHandles,
		ttlCompactFiltersManager);
}
 
Example 22
@Override
public DBOptions createDBOptions(DBOptions currentOptions) {
	if (isOptionConfigured(MAX_BACKGROUND_THREADS)) {
		currentOptions.setIncreaseParallelism(getMaxBackgroundThreads());
	}

	if (isOptionConfigured(MAX_OPEN_FILES)) {
		currentOptions.setMaxOpenFiles(getMaxOpenFiles());
	}

	return currentOptions;
}
 
Example 23
Source Project: Flink-CEPplus   Source File: RocksDBStateBackend.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Gets the RocksDB {@link DBOptions} to be used for all RocksDB instances.
 */
public DBOptions getDbOptions() {
	// initial options from pre-defined profile
	DBOptions opt = getPredefinedOptions().createDBOptions();

	// add user-defined options factory, if specified
	if (optionsFactory != null) {
		opt = optionsFactory.createDBOptions(opt);
	}

	// add necessary default options
	opt = opt.setCreateIfMissing(true);

	return opt;
}
 
Example 24
@Test
public void testSetConfigurableOptions() throws Exception  {
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	assertNull(rocksDbBackend.getOptions());

	DefaultConfigurableOptionsFactory customizedOptions = new DefaultConfigurableOptionsFactory()
		.setMaxBackgroundThreads(4)
		.setMaxOpenFiles(-1)
		.setCompactionStyle(CompactionStyle.LEVEL)
		.setUseDynamicLevelSize(true)
		.setTargetFileSizeBase("4MB")
		.setMaxSizeLevelBase("128 mb")
		.setWriteBufferSize("128 MB")
		.setMaxWriteBufferNumber(4)
		.setMinWriteBufferNumberToMerge(3)
		.setBlockSize("64KB")
		.setBlockCacheSize("512mb");

	rocksDbBackend.setOptions(customizedOptions);

	try (DBOptions dbOptions = rocksDbBackend.getDbOptions()) {
		assertEquals(-1, dbOptions.maxOpenFiles());
	}

	try (ColumnFamilyOptions columnOptions = rocksDbBackend.getColumnOptions()) {
		assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
		assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
		assertEquals(4 * SizeUnit.MB, columnOptions.targetFileSizeBase());
		assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
		assertEquals(4, columnOptions.maxWriteBufferNumber());
		assertEquals(3, columnOptions.minWriteBufferNumberToMerge());

		BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
		assertEquals(64 * SizeUnit.KB, tableConfig.blockSize());
		assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
	}
}
 
Example 25
Source Project: flink   Source File: RocksDBStateBackendConfigTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSetConfigurableOptions() throws Exception  {
	DefaultConfigurableOptionsFactory customizedOptions = new DefaultConfigurableOptionsFactory()
		.setMaxBackgroundThreads(4)
		.setMaxOpenFiles(-1)
		.setCompactionStyle(CompactionStyle.LEVEL)
		.setUseDynamicLevelSize(true)
		.setTargetFileSizeBase("4MB")
		.setMaxSizeLevelBase("128 mb")
		.setWriteBufferSize("128 MB")
		.setMaxWriteBufferNumber(4)
		.setMinWriteBufferNumberToMerge(3)
		.setBlockSize("64KB")
		.setBlockCacheSize("512mb");

	try (RocksDBResourceContainer optionsContainer =
			new RocksDBResourceContainer(PredefinedOptions.DEFAULT, customizedOptions)) {

		DBOptions dbOptions = optionsContainer.getDbOptions();
		assertEquals(-1, dbOptions.maxOpenFiles());

		ColumnFamilyOptions columnOptions = optionsContainer.getColumnOptions();
		assertEquals(CompactionStyle.LEVEL, columnOptions.compactionStyle());
		assertTrue(columnOptions.levelCompactionDynamicLevelBytes());
		assertEquals(4 * SizeUnit.MB, columnOptions.targetFileSizeBase());
		assertEquals(128 * SizeUnit.MB, columnOptions.maxBytesForLevelBase());
		assertEquals(4, columnOptions.maxWriteBufferNumber());
		assertEquals(3, columnOptions.minWriteBufferNumberToMerge());

		BlockBasedTableConfig tableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
		assertEquals(64 * SizeUnit.KB, tableConfig.blockSize());
		assertEquals(512 * SizeUnit.MB, tableConfig.blockCacheSize());
	}
}
 
Example 26
@Test
public void testPredefinedOptionsEnum() {
	for (PredefinedOptions o : PredefinedOptions.values()) {
		try (DBOptions opt = o.createDBOptions()) {
			assertNotNull(opt);
		}
	}
}
 
Example 27
/**
 * Tests list serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testListSerialization()
 * KvStateRequestSerializerTest#testListSerialization() using the heap state back-end
 * test
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(VoidNamespaceSerializer.INSTANCE,
			new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	KvStateRequestSerializerTest.testListSerialization(key, listState);
	longHeapKeyedStateBackend.dispose();
}
 
Example 28
Source Project: sofa-jraft   Source File: StorageOptionsFactory.java    License: Apache License 2.0 5 votes vote down vote up
public static DBOptions getDefaultRocksDBOptions() {
    // Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide
    final DBOptions opts = new DBOptions();

    // If this value is set to true, then the database will be created if it is
    // missing during {@code RocksDB.open()}.
    opts.setCreateIfMissing(true);

    // If true, missing column families will be automatically created.
    opts.setCreateMissingColumnFamilies(true);

    // Number of open files that can be used by the DB.  You may need to increase
    // this if your database has a large working set. Value -1 means files opened
    // are always kept open.
    opts.setMaxOpenFiles(-1);

    // The maximum number of concurrent background compactions. The default is 1,
    // but to fully utilize your CPU and storage you might want to increase this
    // to approximately number of cores in the system.
    opts.setMaxBackgroundCompactions(Math.min(Utils.cpus(), 4));

    // The maximum number of concurrent flush operations. It is usually good enough
    // to set this to 1.
    opts.setMaxBackgroundFlushes(1);

    return opts;
}
 
Example 29
Source Project: flink   Source File: RocksDBResourceContainerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFreeMultipleDBOptionsAfterClose() throws Exception {
	RocksDBResourceContainer container = new RocksDBResourceContainer();
	final int optionNumber = 20;
	ArrayList<DBOptions> dbOptions = new ArrayList<>(optionNumber);
	for (int i = 0; i < optionNumber; i++) {
		dbOptions.add(container.getDbOptions());
	}
	container.close();
	for (DBOptions dbOption: dbOptions) {
		assertThat(dbOption.isOwningHandle(), is(false));
	}
}
 
Example 30
Source Project: flink   Source File: RocksDBOperationUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static RocksDB openDB(
	String path,
	List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors,
	List<ColumnFamilyHandle> stateColumnFamilyHandles,
	ColumnFamilyOptions columnFamilyOptions,
	DBOptions dbOptions) throws IOException {
	List<ColumnFamilyDescriptor> columnFamilyDescriptors =
		new ArrayList<>(1 + stateColumnFamilyDescriptors.size());

	// we add the required descriptor for the default CF in FIRST position, see
	// https://github.com/facebook/rocksdb/wiki/RocksJava-Basics#opening-a-database-with-column-families
	columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions));
	columnFamilyDescriptors.addAll(stateColumnFamilyDescriptors);

	RocksDB dbRef;

	try {
		dbRef = RocksDB.open(
			Preconditions.checkNotNull(dbOptions),
			Preconditions.checkNotNull(path),
			columnFamilyDescriptors,
			stateColumnFamilyHandles);
	} catch (RocksDBException e) {
		IOUtils.closeQuietly(columnFamilyOptions);
		columnFamilyDescriptors.forEach((cfd) -> IOUtils.closeQuietly(cfd.getOptions()));
		throw new IOException("Error while opening RocksDB instance.", e);
	}

	// requested + default CF
	Preconditions.checkState(1 + stateColumnFamilyDescriptors.size() == stateColumnFamilyHandles.size(),
		"Not all requested column family handles have been created");
	return dbRef;
}