org.rocksdb.TtlDB Java Examples

RocksDBStoreImpl(String path, int ttl, int maxCompactions) {
    try {
        ensureDirectoryExists(path);

        final Options options = new Options();
        options.setCreateIfMissing(true);
        options.setMaxBackgroundCompactions(maxCompactions);
        options.setMaxOpenFiles(2);//RocksDB 会将打开的 SST 文件句柄缓存这，这样下次访问的时候就可以直接使用，而不需要重新在打开。
        options.setWriteBufferSize(4 * MB_BYTE);//4M, memtable 的最大 size
        options.setMaxWriteBufferNumber(4);//最大 memtable 的个数
        options.setLevel0FileNumCompactionTrigger(4);//当有4个未进行Compact的文件时，达到触发Compact的条件

        this.rocksDB = TtlDB.open(options, path, ttl, false);
        LOG.info("open rocks db success, path:{}, ttl:{}", path, ttl);

    } catch (Exception e) {
        LOG.error("open rocks db error, path:{}, ttl:{}", path, ttl, e);
        throw new RuntimeException(e);
    }
}
 

private static TtlDB getTtlDb() throws Exception {
    final String path = "/home/test/rocsksdb";
    ensureDirectoryExists(path);
    final Options options = new Options();
    final int ttl = (int) TimeUnit.HOURS.toSeconds(1);
    options.setCreateIfMissing(true);
    options.setMaxBackgroundCompactions(2);
    options.setMaxOpenFiles(2);//RocksDB 会将打开的 SST 文件句柄缓存这，这样下次访问的时候就可以直接使用，而不需要重新在打开。
    options.setWriteBufferSize(4194304);//4M, memtable 的最大 size
    options.setMaxWriteBufferNumber(4);//最大 memtable 的个数
    options.setLevel0FileNumCompactionTrigger(2);//当有4个未进行Compact的文件时，达到触发Compact的条件
    //options.setMaxCompactionBytes(0);
    RemoveEmptyValueCompactionFilter filter = new RemoveEmptyValueCompactionFilter();
    final TtlDB ttlDB = TtlDB.open(options, path, ttl, false);
    return ttlDB;
}
 

public RocksDBStoreImpl(final DynamicConfig config) {
    final String path = config.getString(ROCKS_DB_PATH_CONFIG_KEY);
    final int ttl = config.getInt(ROCKS_DB_TTL_CONFIG_KEY, DEFAULT_ROCKS_DB_TTL);

    File file = new File(path);
    if (!file.exists() || !file.isDirectory()) {
        if (!file.mkdirs()) {
            throw new RuntimeException("Failed to create RocksDB dir.");
        }
    }

    try {
        final Options options = new Options();
        options.setCreateIfMissing(true);
        this.rocksDB = TtlDB.open(options, path, ttl, false);
        LOG.info("open rocks db success, path:{}, ttl:{}", path, ttl);
    } catch (Exception e) {
        LOG.error("open rocks db error, path:{}, ttl:{}", path, ttl, e);
        throw new RuntimeException(e);
    }
}
 

protected void initRocksDb() {
    RocksDbOptionsFactory optionFactory = new RocksDbOptionsFactory.Defaults();
    Options options = optionFactory.createOptions(null);
    DBOptions dbOptions = optionFactory.createDbOptions(null);
    ColumnFamilyOptions cfOptions = optionFactory.createColumnFamilyOptions(null);
    String optionsFactoryClass = (String) conf.get(ConfigExtension.ROCKSDB_OPTIONS_FACTORY_CLASS);
    if (optionsFactoryClass != null) {
        RocksDbOptionsFactory udfOptionFactory = (RocksDbOptionsFactory) Utils.newInstance(optionsFactoryClass);
        options = udfOptionFactory.createOptions(options);
        dbOptions = udfOptionFactory.createDbOptions(dbOptions);
        cfOptions = udfOptionFactory.createColumnFamilyOptions(cfOptions);
    }

    try {
        ttlTimeSec = ConfigExtension.getStateTtlTime(conf);
        if (ttlTimeSec > 0)
            rocksDb = TtlDB.open(options, rocksDbDir, ttlTimeSec, false);
        else
            rocksDb = RocksDB.open(options, rocksDbDir);
        // enable compaction
        rocksDb.compactRange();
        LOG.info("Finish the initialization of RocksDB");
    } catch (RocksDBException e) {
        LOG.error("Failed to open rocksdb located at " + rocksDbDir, e);
        throw new RuntimeException(e.getMessage());
    }

    lastCheckpointFiles = new HashSet<String>();
    lastCleanTime = System.currentTimeMillis();
    lastSuccessBatchId = -1;
}
 

public void initDb(List<Integer> list) throws Exception {
    LOG.info("Begin to init rocksDB of {}", rootDir);

    DBOptions dbOptions = null;

    List<ColumnFamilyDescriptor> columnFamilyNames = new ArrayList<>();
    columnFamilyNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
    for (Integer timeout : list) {
        columnFamilyNames.add(new ColumnFamilyDescriptor(String.valueOf(timeout).getBytes()));
    }

    List<Integer> ttlValues = new ArrayList<>();
    // Default column family with infinite TTL
    // NOTE that the first must be 0, RocksDB.java API has this limitation
    ttlValues.add(0);
    // new column family with list second ttl
    ttlValues.addAll(list);

    try {
        dbOptions = new DBOptions().setCreateMissingColumnFamilies(true).setCreateIfMissing(true);
        List<ColumnFamilyHandle> columnFamilyHandleList = new ArrayList<>();
        ttlDB = TtlDB.open(dbOptions, rootDir, columnFamilyNames, columnFamilyHandleList, ttlValues, false);
        for (int i = 0; i < ttlValues.size(); i++) {
            windowHandlers.put(ttlValues.get(i), columnFamilyHandleList.get(i));
        }
        LOG.info("Successfully init rocksDB of {}", rootDir);
    } finally {

        if (dbOptions != null) {
            dbOptions.dispose();
        }
    }
}
 

@Override
protected void initRocksDb() {
    windowToCFHandler = new HashMap<>();

    RocksDbOptionsFactory optionFactory = new RocksDbOptionsFactory.Defaults();
    Options options = optionFactory.createOptions(null);
    DBOptions dbOptions = optionFactory.createDbOptions(null);
    ColumnFamilyOptions cfOptions = optionFactory.createColumnFamilyOptions(null);
    String optionsFactoryClass = (String) conf.get(ConfigExtension.ROCKSDB_OPTIONS_FACTORY_CLASS);
    if (optionsFactoryClass != null) {
        RocksDbOptionsFactory udfOptionFactory = (RocksDbOptionsFactory) Utils.newInstance(optionsFactoryClass);
        options = udfOptionFactory.createOptions(options);
        dbOptions = udfOptionFactory.createDbOptions(dbOptions);
        cfOptions = udfOptionFactory.createColumnFamilyOptions(cfOptions);
    }

    try {
        ttlTimeSec = ConfigExtension.getStateTtlTime(conf);
        List<Integer> ttlValues = new ArrayList<>();

        List<byte[]> families = RocksDB.listColumnFamilies(options, rocksDbDir);
        List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
        List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
        if (families != null) {
            for (byte[] bytes : families) {
                columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, cfOptions));
                LOG.debug("Load colum family of {}", new String(bytes));
                if (ttlTimeSec > 0)
                    ttlValues.add(ttlTimeSec);
            }
        }
        
        if (columnFamilyDescriptors.size() > 0) {
            if (ttlTimeSec > 0)
                rocksDb = TtlDB.open(dbOptions, rocksDbDir, columnFamilyDescriptors, columnFamilyHandles, ttlValues, false);
            else
                rocksDb = RocksDB.open(dbOptions, rocksDbDir, columnFamilyDescriptors, columnFamilyHandles);

            int n = Math.min(columnFamilyDescriptors.size(), columnFamilyHandles.size());
            LOG.info("Try to load RocksDB with column family, desc_num={}, handler_num={}", columnFamilyDescriptors.size(), columnFamilyHandles.size());
            // skip default column
            for (int i = 1; i < n; i++) {
                windowToCFHandler.put((TimeWindow) serializer.deserialize(columnFamilyDescriptors.get(i).columnFamilyName()), columnFamilyHandles.get(i));
            }
        } else {
            rocksDb = RocksDB.open(options, rocksDbDir);
        }
        rocksDb.compactRange();
        LOG.info("Finish the initialization of RocksDB");
    } catch (RocksDBException e) {
        LOG.error("Failed to open rocksdb located at " + rocksDbDir, e);
        throw new RuntimeException(e.getMessage());
    }

    lastCheckpointFiles = new HashSet<String>();
    lastCleanTime = System.currentTimeMillis();
    lastSuccessBatchId = -1;
}
 

public void ttlDbOpenWithColumnFamilies() throws RocksDBException, InterruptedException {
    DBOptions dbOptions = null;
    TtlDB ttlDB = null;
    List<ColumnFamilyDescriptor> cfNames = new ArrayList<ColumnFamilyDescriptor>();
    List<ColumnFamilyHandle> columnFamilyHandleList = new ArrayList<ColumnFamilyHandle>();
    cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
    cfNames.add(new ColumnFamilyDescriptor("new_cf".getBytes()));

    List<Integer> ttlValues = new ArrayList<Integer>();
    // Default column family with infinite lifetime
    ttlValues.add(0);
    // new column family with 1 second ttl
    ttlValues.add(1);

    try {
        System.out.println("Begin to open db");
        dbOptions = new DBOptions().setCreateMissingColumnFamilies(true).setCreateIfMissing(true);
        ttlDB = TtlDB.open(dbOptions, rootDir, cfNames, columnFamilyHandleList, ttlValues, false);

        System.out.println("Successfully open db " + rootDir);

        ttlDB.put("key".getBytes(), "value".getBytes());
        assertThat(ttlDB.get("key".getBytes())).isEqualTo("value".getBytes());
        ttlDB.put(columnFamilyHandleList.get(1), "key".getBytes(), "value".getBytes());
        assertThat(ttlDB.get(columnFamilyHandleList.get(1), "key".getBytes())).isEqualTo("value".getBytes());
        TimeUnit.SECONDS.sleep(2);

        ttlDB.compactRange();
        ttlDB.compactRange(columnFamilyHandleList.get(1));

        assertThat(ttlDB.get("key".getBytes())).isNotNull();
        assertThat(ttlDB.get(columnFamilyHandleList.get(1), "key".getBytes())).isNull();

    } finally {
        for (ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) {
            columnFamilyHandle.dispose();
        }
        if (ttlDB != null) {
            ttlDB.close();
        }
        if (dbOptions != null) {
            dbOptions.dispose();
        }
    }
}