Java Code Examples for org.rocksdb.DBOptions#setCreateIfMissing()

The following examples show how to use org.rocksdb.DBOptions#setCreateIfMissing() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestTypedRDBTableStore.java    From hadoop-ozone with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);

  Set<TableConfig> configSet = new HashSet<>();
  for (String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);

  codecRegistry = new CodecRegistry();

}
 
Example 2
Source File: TestRDBStore.java    From hadoop-ozone with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);
  configSet = new HashSet<>();
  for(String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);
}
 
Example 3
Source File: TestRDBTableStore.java    From hadoop-ozone with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  options = new DBOptions();
  options.setCreateIfMissing(true);
  options.setCreateMissingColumnFamilies(true);

  Statistics statistics = new Statistics();
  statistics.setStatsLevel(StatsLevel.ALL);
  options = options.setStatistics(statistics);

  Set<TableConfig> configSet = new HashSet<>();
  for(String name : families) {
    TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
    configSet.add(newConfig);
  }
  rdbStore = new RDBStore(folder.newFolder(), options, configSet);
}
 
Example 4
Source File: SQLConnection.java    From act with GNU General Public License v3.0 6 votes vote down vote up
public Pair<RocksDB, Map<String, ColumnFamilyHandle>> openSupportingIndex(File supportingIndex)
    throws RocksDBException {
  List<FromBrendaDB> instances = BrendaSupportingEntries.allFromBrendaDBInstances();
  List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(instances.size() + 1);
  columnFamilyDescriptors.add(new ColumnFamilyDescriptor("default".getBytes()));
  for (FromBrendaDB instance : instances) {
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(instance.getColumnFamilyName().getBytes()));
  }
  List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size());

  DBOptions dbOptions = new DBOptions();
  dbOptions.setCreateIfMissing(false);
  RocksDB rocksDB = RocksDB.open(dbOptions, supportingIndex.getAbsolutePath(),
      columnFamilyDescriptors, columnFamilyHandles);
  Map<String, ColumnFamilyHandle> columnFamilyHandleMap = new HashMap<>(columnFamilyHandles.size());
  // TODO: can we zip these together more easily w/ Java 8?

  for (int i = 0; i < columnFamilyDescriptors.size(); i++) {
    ColumnFamilyDescriptor cfd = columnFamilyDescriptors.get(i);
    ColumnFamilyHandle cfh = columnFamilyHandles.get(i);
    columnFamilyHandleMap.put(new String(cfd.columnFamilyName(), BrendaSupportingEntries.UTF8), cfh);
  }

  return Pair.of(rocksDB, columnFamilyHandleMap);
}
 
Example 5
Source File: RocksDBResourceContainer.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Gets the RocksDB {@link DBOptions} to be used for RocksDB instances.
 */
public DBOptions getDbOptions() {
	// initial options from pre-defined profile
	DBOptions opt = predefinedOptions.createDBOptions(handlesToClose);
	handlesToClose.add(opt);

	// add user-defined options factory, if specified
	if (optionsFactory != null) {
		opt = optionsFactory.createDBOptions(opt, handlesToClose);
	}

	// add necessary default options
	opt = opt.setCreateIfMissing(true);

	// if sharedResources is non-null, use the write buffer manager from it.
	if (sharedResources != null) {
		opt.setWriteBufferManager(sharedResources.getResourceHandle().getWriteBufferManager());
	}

	return opt;
}
 
Example 6
Source File: RocksDBStateBackend.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Gets the RocksDB {@link DBOptions} to be used for all RocksDB instances.
 */
public DBOptions getDbOptions() {
	// initial options from pre-defined profile
	DBOptions opt = getPredefinedOptions().createDBOptions();

	// add user-defined options factory, if specified
	if (optionsFactory != null) {
		opt = optionsFactory.createDBOptions(opt);
	}

	// add necessary default options
	opt = opt.setCreateIfMissing(true);

	return opt;
}
 
Example 7
Source File: KVStateRequestSerializerRocksDBTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests list serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testListSerialization()
 * KvStateRequestSerializerTest#testListSerialization() using the heap state back-end
 * test
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(VoidNamespaceSerializer.INSTANCE,
			new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	KvStateRequestSerializerTest.testListSerialization(key, listState);
	longHeapKeyedStateBackend.dispose();
}
 
Example 8
Source File: StorageOptionsFactory.java    From sofa-jraft with Apache License 2.0 5 votes vote down vote up
public static DBOptions getDefaultRocksDBOptions() {
    // Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide
    final DBOptions opts = new DBOptions();

    // If this value is set to true, then the database will be created if it is
    // missing during {@code RocksDB.open()}.
    opts.setCreateIfMissing(true);

    // If true, missing column families will be automatically created.
    opts.setCreateMissingColumnFamilies(true);

    // Number of open files that can be used by the DB.  You may need to increase
    // this if your database has a large working set. Value -1 means files opened
    // are always kept open.
    opts.setMaxOpenFiles(-1);

    // The maximum number of concurrent background compactions. The default is 1,
    // but to fully utilize your CPU and storage you might want to increase this
    // to approximately number of cores in the system.
    opts.setMaxBackgroundCompactions(Math.min(Utils.cpus(), 4));

    // The maximum number of concurrent flush operations. It is usually good enough
    // to set this to 1.
    opts.setMaxBackgroundFlushes(1);

    return opts;
}
 
Example 9
Source File: RocksDBStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Gets the RocksDB {@link DBOptions} to be used for all RocksDB instances.
 */
public DBOptions getDbOptions() {
	// initial options from pre-defined profile
	DBOptions opt = getPredefinedOptions().createDBOptions();

	// add user-defined options factory, if specified
	if (optionsFactory != null) {
		opt = optionsFactory.createDBOptions(opt);
	}

	// add necessary default options
	opt = opt.setCreateIfMissing(true);

	return opt;
}
 
Example 10
Source File: KVStateRequestSerializerRocksDBTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests list serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testListSerialization()
 * KvStateRequestSerializerTest#testListSerialization() using the heap state back-end
 * test
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(VoidNamespaceSerializer.INSTANCE,
			new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	KvStateRequestSerializerTest.testListSerialization(key, listState);
	longHeapKeyedStateBackend.dispose();
}
 
Example 11
Source File: PubchemTTLMerger.java    From act with GNU General Public License v3.0 5 votes vote down vote up
/**
 * Open an existing RocksDB index.  Use this after successful index generation to access the map of Pubchem compound
 * ids to synonyms/MeSH ids using the column family CID_TO_SYNONYMS.
 * @param pathToIndex A path to the RocksDB index directory to use.
 * @return
 * @throws RocksDBException
 */
public static Pair<RocksDB, Map<COLUMN_FAMILIES, ColumnFamilyHandle>> openExistingRocksDB(File pathToIndex)
    throws RocksDBException {
  List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(COLUMN_FAMILIES.values().length + 1);
  // Must also open the "default" family or RocksDB will probably choke.
  columnFamilyDescriptors.add(new ColumnFamilyDescriptor(DEFAULT_ROCKSDB_COLUMN_FAMILY.getBytes()));
  for (COLUMN_FAMILIES family : COLUMN_FAMILIES.values()) {
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(family.getName().getBytes()));
  }
  List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size());

  DBOptions dbOptions = ROCKS_DB_OPEN_OPTIONS;
  dbOptions.setCreateIfMissing(false);
  RocksDB rocksDB = RocksDB.open(dbOptions, pathToIndex.getAbsolutePath(),
      columnFamilyDescriptors, columnFamilyHandles);
  Map<COLUMN_FAMILIES, ColumnFamilyHandle> columnFamilyHandleMap = new HashMap<>(COLUMN_FAMILIES.values().length);
  // TODO: can we zip these together more easily w/ Java 8?

  for (int i = 0; i < columnFamilyDescriptors.size(); i++) {
    ColumnFamilyDescriptor cfd = columnFamilyDescriptors.get(i);
    ColumnFamilyHandle cfh = columnFamilyHandles.get(i);
    String familyName = new String(cfd.columnFamilyName(), UTF8);
    COLUMN_FAMILIES descriptorFamily = COLUMN_FAMILIES.getFamilyByName(familyName);
    if (descriptorFamily == null) {
      if (!DEFAULT_ROCKSDB_COLUMN_FAMILY.equals(familyName)) {
        String msg = String.format("Found unexpected family name '%s' when trying to open RocksDB at %s",
            familyName, pathToIndex.getAbsolutePath());
        LOGGER.error(msg);
        // Crash if we don't recognize the contents of this DB.
        throw new RuntimeException(msg);
      }
      // Just skip this column family if it doesn't map to something we know but is expected.
      continue;
    }

    columnFamilyHandleMap.put(descriptorFamily, cfh);
  }

  return Pair.of(rocksDB, columnFamilyHandleMap);
}
 
Example 12
Source File: KVStateRequestSerializerRocksDBTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Tests map serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testMapSerialization()
 * KvStateRequestSerializerTest#testMapSerialization() using the heap state back-end
 * test
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	KvStateRequestSerializerTest.testMapSerialization(key, mapState);
	longHeapKeyedStateBackend.dispose();
}
 
Example 13
Source File: KVStateRequestSerializerRocksDBTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests map serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testMapSerialization()
 * KvStateRequestSerializerTest#testMapSerialization() using the heap state back-end
 * test
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	KvStateRequestSerializerTest.testMapSerialization(key, mapState);
	longHeapKeyedStateBackend.dispose();
}
 
Example 14
Source File: DBUtil.java    From act with GNU General Public License v3.0 4 votes vote down vote up
/**
 * Open an existing RocksDB index.
 * @param pathToIndex A path to the RocksDB index directory to use.
 * @param columnFamilies A list of column familities to open.  Must be exhaustive, non-empty, and non-null.
 * @return A DB and map of column family labels (as T) to enums.
 * @throws RocksDBException
 */
public static <T extends ColumnFamilyEnumeration<T>> RocksDBAndHandles<T> openExistingRocksDB(
    File pathToIndex, T[] columnFamilies) throws RocksDBException {
  if (columnFamilies == null || columnFamilies.length == 0) {
    throw new RuntimeException("Cannot open a RocksDB with an empty list of column families.");
  }

  List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(columnFamilies.length + 1);
  // Must also open the "default" family or RocksDB will probably choke.
  columnFamilyDescriptors.add(new ColumnFamilyDescriptor(DEFAULT_ROCKSDB_COLUMN_FAMILY.getBytes()));
  for (T family : columnFamilies) {
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(family.getName().getBytes()));
  }
  List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size());

  DBOptions dbOptions = ROCKS_DB_OPEN_OPTIONS;
  dbOptions.setCreateIfMissing(false);
  RocksDB db = RocksDB.open(dbOptions, pathToIndex.getAbsolutePath(),
      columnFamilyDescriptors, columnFamilyHandles);
  Map<T, ColumnFamilyHandle> columnFamilyHandleMap = new HashMap<>(columnFamilies.length);
  // TODO: can we zip these together more easily w/ Java 8?

  for (int i = 0; i < columnFamilyDescriptors.size(); i++) {
    ColumnFamilyDescriptor cfd = columnFamilyDescriptors.get(i);
    ColumnFamilyHandle cfh = columnFamilyHandles.get(i);
    String familyName = new String(cfd.columnFamilyName(), UTF8);
    T descriptorFamily = columnFamilies[0].getFamilyByName(familyName); // Use any instance to get the next family.
    if (descriptorFamily == null) {
      if (!DEFAULT_ROCKSDB_COLUMN_FAMILY.equals(familyName)) {
        String msg = String.format("Found unexpected family name '%s' when trying to open RocksDB at %s",
            familyName, pathToIndex.getAbsolutePath());
        LOGGER.error(msg);
        // Crash if we don't recognize the contents of this DB.
        throw new RuntimeException(msg);
      }
      // Just skip this column family if it doesn't map to something we know but is expected.
      continue;
    }

    columnFamilyHandleMap.put(descriptorFamily, cfh);
  }

  return new RocksDBAndHandles<T>(db, columnFamilyHandleMap);
}
 
Example 15
Source File: RocksDbUnitTest.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) {
    Map conf = JStormHelper.LoadConf(args[0]);
    putNum = JStormUtils.parseInt(conf.get("put.number"), 100);
    isFlush = JStormUtils.parseBoolean(conf.get("is.flush"), true);
    isCheckpoint = JStormUtils.parseBoolean(conf.get("is.checkpoint"), true);
    sleepTime = JStormUtils.parseInt(conf.get("sleep.time"), 5000);
    compactionInterval = JStormUtils.parseInt(conf.get("compaction.interval"), 30000);
    flushInterval = JStormUtils.parseInt(conf.get("flush.interval"), 3000);
    isCompaction = JStormUtils.parseBoolean(conf.get("is.compaction"), true);
    fileSizeBase = JStormUtils.parseLong(conf.get("file.size.base"), 10 * SizeUnit.KB);
    levelNum = JStormUtils.parseInt(conf.get("db.level.num"), 1);
    compactionTriggerNum = JStormUtils.parseInt(conf.get("db.compaction.trigger.num"), 4);
    LOG.info("Conf={}", conf);
    
    RocksDB db;
    File file = new File(cpPath);
    file.mkdirs();

    List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
    try {
        Options options = new Options();
        options.setCreateMissingColumnFamilies(true);
        options.setCreateIfMissing(true);
        options.setTargetFileSizeBase(fileSizeBase);
        options.setMaxBackgroundFlushes(2);
        options.setMaxBackgroundCompactions(2);
        options.setCompactionStyle(CompactionStyle.LEVEL);
        options.setNumLevels(levelNum);
        options.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);

        DBOptions dbOptions = new DBOptions();
        dbOptions.setCreateMissingColumnFamilies(true);
        dbOptions.setCreateIfMissing(true);
        dbOptions.setMaxBackgroundFlushes(2);
        dbOptions.setMaxBackgroundCompactions(2);
        ColumnFamilyOptions familyOptions = new ColumnFamilyOptions();
        familyOptions.setTargetFileSizeBase(fileSizeBase);
        familyOptions.setCompactionStyle(CompactionStyle.LEVEL);
        familyOptions.setNumLevels(levelNum);
        familyOptions.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);
        List<byte[]> families = RocksDB.listColumnFamilies(options, dbPath);
        List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
        if (families != null) {
            for (byte[] bytes : families) {
                columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, familyOptions));
                LOG.info("Load colum family of {}", new String(bytes));
            }
        }
        
        if (columnFamilyDescriptors.size() > 0) {
            db = RocksDB.open(dbOptions, dbPath, columnFamilyDescriptors, columnFamilyHandles);
        } else {
            db = RocksDB.open(options, dbPath);
        }
    } catch (RocksDBException e) {
        LOG.error("Failed to open db", e);
        return;
    }

    rocksDbTest(db, columnFamilyHandles);
    
    db.close();
}