org.rocksdb.CompressionType Java Examples

The following examples show how to use org.rocksdb.CompressionType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KnowledgeBase.java    From fasten with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("resource")
public static KnowledgeBase getInstance(final String kbDir, final String kbMetadataPathname, final boolean readOnly) throws RocksDBException, ClassNotFoundException, IOException {
	final boolean metadataExists = new File(kbMetadataPathname).exists();
	final boolean kbDirExists = new File(kbDir).exists();
	if (metadataExists != kbDirExists) throw new IllegalArgumentException("Either both or none of the knowledge-base directory and metadata must exist");

	RocksDB.loadLibrary();
	final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions().setCompressionType(CompressionType.LZ4_COMPRESSION);
	final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true);
	final List<ColumnFamilyDescriptor> cfDescriptors = Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions), new ColumnFamilyDescriptor(GID2URI, cfOptions), new ColumnFamilyDescriptor(URI2GID, cfOptions));

	final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
	final RocksDB db = readOnly ? RocksDB.openReadOnly(dbOptions, kbDir, cfDescriptors, columnFamilyHandles) : RocksDB.open(dbOptions, kbDir, cfDescriptors, columnFamilyHandles);

	final KnowledgeBase kb;
	if (metadataExists) {
		kb = (KnowledgeBase) BinIO.loadObject(kbMetadataPathname);
		kb.readOnly = readOnly;
		kb.callGraphDB = db;
		kb.defaultHandle = columnFamilyHandles.get(0);
		kb.gid2uriFamilyHandle = columnFamilyHandles.get(1);
		kb.uri2gidFamilyHandle = columnFamilyHandles.get(2);
	} else kb = new KnowledgeBase(db, columnFamilyHandles.get(0), columnFamilyHandles.get(1), columnFamilyHandles.get(2), kbMetadataPathname, readOnly);
	return kb;
}
 
Example #2
Source File: RocksDBLookupBuilder.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public RocksDBLookupBuilder(TableDesc tableDesc, String[] keyColumns, String dbPath) {
    this.tableDesc = tableDesc;
    this.encoder = new RocksDBLookupRowEncoder(tableDesc, keyColumns);
    this.dbPath = dbPath;
    this.writeBatchSize = 500;
    this.options = new Options();
    options.setCreateIfMissing(true).setWriteBufferSize(8 * SizeUnit.KB).setMaxWriteBufferNumber(3)
            .setMaxBackgroundCompactions(5).setCompressionType(CompressionType.SNAPPY_COMPRESSION)
            .setCompactionStyle(CompactionStyle.UNIVERSAL);

}
 
Example #3
Source File: RocksDBWrapper.java    From aion with MIT License 5 votes vote down vote up
private Options setupRocksDbOptions() {
    Options options = new Options();

    options.setCreateIfMissing(true);
    options.setUseFsync(false);
    options.setCompressionType(
            enableDbCompression
                    ? CompressionType.LZ4_COMPRESSION
                    : CompressionType.NO_COMPRESSION);

    options.setBottommostCompressionType(CompressionType.ZLIB_COMPRESSION);
    options.setMinWriteBufferNumberToMerge(MIN_WRITE_BUFFER_NUMBER_TOMERGE);
    options.setLevel0StopWritesTrigger(LEVEL0_STOP_WRITES_TRIGGER);
    options.setLevel0SlowdownWritesTrigger(LEVEL0_SLOWDOWN_WRITES_TRIGGER);
    options.setAtomicFlush(true);
    options.setWriteBufferSize(this.writeBufferSize);
    options.setRandomAccessMaxBufferSize(this.readBufferSize);
    options.setParanoidChecks(true);
    options.setMaxOpenFiles(this.maxOpenFiles);
    options.setTableFormatConfig(setupBlockBasedTableConfig());
    options.setDisableAutoCompactions(false);
    options.setIncreaseParallelism(max(1, Runtime.getRuntime().availableProcessors() / 2));

    options.setLevelCompactionDynamicLevelBytes(true);
    options.setMaxBackgroundCompactions(MAX_BACKGROUND_COMPACTIONS);
    options.setMaxBackgroundFlushes(MAX_BACKGROUND_FLUSHES);
    options.setBytesPerSync(BYTES_PER_SYNC);
    options.setCompactionPriority(CompactionPriority.MinOverlappingRatio);
    options.optimizeLevelStyleCompaction(OPTIMIZE_LEVEL_STYLE_COMPACTION);

    return options;
}
 
Example #4
Source File: RocksDBLookupBuilder.java    From kylin with Apache License 2.0 5 votes vote down vote up
public RocksDBLookupBuilder(TableDesc tableDesc, String[] keyColumns, String dbPath) {
    this.tableDesc = tableDesc;
    this.encoder = new RocksDBLookupRowEncoder(tableDesc, keyColumns);
    this.dbPath = dbPath;
    this.writeBatchSize = 500;
    this.options = new Options();
    options.setCreateIfMissing(true).setWriteBufferSize(8 * SizeUnit.KB).setMaxWriteBufferNumber(3)
            .setMaxBackgroundCompactions(5).setCompressionType(CompressionType.SNAPPY_COMPRESSION)
            .setCompactionStyle(CompactionStyle.UNIVERSAL);

}
 
Example #5
Source File: RocksDBConfigParser.java    From journalkeeper with Apache License 2.0 4 votes vote down vote up
public static Options parse(Properties properties) {
    Options options = new Options();
    options.setCompressionType(CompressionType.LZ4_COMPRESSION)
            .setCompactionStyle(CompactionStyle.LEVEL);

    BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
    options.setTableFormatConfig(tableOptions);

    for (String key : properties.stringPropertyNames()) {
        String prefix = null;
        Object configInstance = null;

        if (key.startsWith(RocksDBConfigs.OPTIONS_PREFIX)) {
            prefix = RocksDBConfigs.OPTIONS_PREFIX;
            configInstance = options;
        } else if (key.startsWith(RocksDBConfigs.TABLE_OPTIONS_PREFIX)) {
            prefix = RocksDBConfigs.TABLE_OPTIONS_PREFIX;
            configInstance = tableOptions;
        } else {
            continue;
        }

        String fieldKey = key.substring(prefix.length(), key.length());
        String value = properties.getProperty(key);

        try {
            Method setterMethod = findSetterMethod(configInstance.getClass(), fieldKey);
            if (setterMethod == null) {
                logger.warn("parse config error, method not found, key: {}, value: {}", key, value);
                continue;
            }
            setterMethod.invoke(configInstance, PropertyUtils.convert(value, setterMethod.getParameters()[0].getType()));
        } catch (Exception e) {
            logger.error("parse config error, key: {}, value: {}", key, value, e);
        }
    }

    if (properties.containsKey(RocksDBConfigs.FILTER_BITSPER_KEY)) {
        tableOptions.setFilterPolicy(new BloomFilter(
                PropertyUtils.convertInt(properties.getProperty(RocksDBConfigs.FILTER_BITSPER_KEY), 0)));
    }

    return options;
}
 
Example #6
Source File: StorageOptionsFactory.java    From sofa-jraft with Apache License 2.0 4 votes vote down vote up
public static ColumnFamilyOptions getDefaultRocksDBColumnFamilyOptions() {
    final ColumnFamilyOptions opts = new ColumnFamilyOptions();

    // Flushing options:
    // write_buffer_size sets the size of a single mem_table. Once mem_table exceeds
    // this size, it is marked immutable and a new one is created.
    opts.setWriteBufferSize(64 * SizeUnit.MB);

    // Flushing options:
    // max_write_buffer_number sets the maximum number of mem_tables, both active
    // and immutable.  If the active mem_table fills up and the total number of
    // mem_tables is larger than max_write_buffer_number we stall further writes.
    // This may happen if the flush process is slower than the write rate.
    opts.setMaxWriteBufferNumber(3);

    // Flushing options:
    // min_write_buffer_number_to_merge is the minimum number of mem_tables to be
    // merged before flushing to storage. For example, if this option is set to 2,
    // immutable mem_tables are only flushed when there are two of them - a single
    // immutable mem_table will never be flushed.  If multiple mem_tables are merged
    // together, less data may be written to storage since two updates are merged to
    // a single key. However, every Get() must traverse all immutable mem_tables
    // linearly to check if the key is there. Setting this option too high may hurt
    // read performance.
    opts.setMinWriteBufferNumberToMerge(1);

    // Level Style Compaction:
    // level0_file_num_compaction_trigger -- Once level 0 reaches this number of
    // files, L0->L1 compaction is triggered. We can therefore estimate level 0
    // size in stable state as
    // write_buffer_size * min_write_buffer_number_to_merge * level0_file_num_compaction_trigger.
    opts.setLevel0FileNumCompactionTrigger(10);

    // Soft limit on number of level-0 files. We start slowing down writes at this
    // point. A value 0 means that no writing slow down will be triggered by number
    // of files in level-0.
    opts.setLevel0SlowdownWritesTrigger(20);

    // Maximum number of level-0 files.  We stop writes at this point.
    opts.setLevel0StopWritesTrigger(40);

    // Level Style Compaction:
    // max_bytes_for_level_base and max_bytes_for_level_multiplier
    //  -- max_bytes_for_level_base is total size of level 1. As mentioned, we
    // recommend that this be around the size of level 0. Each subsequent level
    // is max_bytes_for_level_multiplier larger than previous one. The default
    // is 10 and we do not recommend changing that.
    opts.setMaxBytesForLevelBase(512 * SizeUnit.MB);

    // Level Style Compaction:
    // target_file_size_base and target_file_size_multiplier
    //  -- Files in level 1 will have target_file_size_base bytes. Each next
    // level's file size will be target_file_size_multiplier bigger than previous
    // one. However, by default target_file_size_multiplier is 1, so files in all
    // L1..LMax levels are equal. Increasing target_file_size_base will reduce total
    // number of database files, which is generally a good thing. We recommend setting
    // target_file_size_base to be max_bytes_for_level_base / 10, so that there are
    // 10 files in level 1.
    opts.setTargetFileSizeBase(64 * SizeUnit.MB);

    // If prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0,
    // create prefix bloom for memtable with the size of
    // write_buffer_size * memtable_prefix_bloom_size_ratio.
    // If it is larger than 0.25, it is santinized to 0.25.
    opts.setMemtablePrefixBloomSizeRatio(0.125);

    // Seems like the rocksDB jni for Windows doesn't come linked with any of the
    // compression type
    if (!Platform.isWindows()) {
        opts.setCompressionType(CompressionType.LZ4_COMPRESSION) //
            .setCompactionStyle(CompactionStyle.LEVEL) //
            .optimizeLevelStyleCompaction();
    }

    // https://github.com/facebook/rocksdb/pull/5744
    opts.setForceConsistencyChecks(true);

    return opts;
}
 
Example #7
Source File: RocksDBStorageEngine.java    From HaloDB with Apache License 2.0 4 votes vote down vote up
@Override
public void open() {
    options = new Options().setCreateIfMissing(true);
    options.setStatsDumpPeriodSec(1000000);

    options.setWriteBufferSize(128l * 1024 * 1024);
    options.setMaxWriteBufferNumber(3);
    options.setMaxBackgroundCompactions(20);

    Env env = Env.getDefault();
    env.setBackgroundThreads(20, Env.COMPACTION_POOL);
    options.setEnv(env);

    // max size of L1 10 MB.
    options.setMaxBytesForLevelBase(10485760);
    options.setTargetFileSizeBase(67108864);

    options.setLevel0FileNumCompactionTrigger(4);
    options.setLevel0SlowdownWritesTrigger(6);
    options.setLevel0StopWritesTrigger(12);
    options.setNumLevels(6);
    options.setDeleteObsoleteFilesPeriodMicros(300000000);


    options.setAllowMmapReads(false);
    options.setCompressionType(CompressionType.SNAPPY_COMPRESSION);


    System.out.printf("maxBackgroundCompactions %d \n", options.maxBackgroundCompactions());
    System.out.printf("minWriteBufferNumberToMerge %d \n", options.minWriteBufferNumberToMerge());
    System.out.printf("maxWriteBufferNumberToMaintain %d \n", options.maxWriteBufferNumberToMaintain());


    System.out.printf("level0FileNumCompactionTrigger %d \n", options.level0FileNumCompactionTrigger());
    System.out.printf("maxBytesForLevelBase %d \n", options.maxBytesForLevelBase());
    System.out.printf("maxBytesForLevelMultiplier %f \n", options.maxBytesForLevelMultiplier());
    System.out.printf("targetFileSizeBase %d \n", options.targetFileSizeBase());
    System.out.printf("targetFileSizeMultiplier %d \n", options.targetFileSizeMultiplier());

    List<CompressionType> compressionLevels =
        Arrays.asList(
            CompressionType.NO_COMPRESSION,
            CompressionType.NO_COMPRESSION,
            CompressionType.SNAPPY_COMPRESSION,
            CompressionType.SNAPPY_COMPRESSION,
            CompressionType.SNAPPY_COMPRESSION,
            CompressionType.SNAPPY_COMPRESSION
        );

    options.setCompressionPerLevel(compressionLevels);

    System.out.printf("compressionPerLevel %s \n", options.compressionPerLevel());
    System.out.printf("numLevels %s \n", options.numLevels());

    writeOptions = new WriteOptions();
    writeOptions.setDisableWAL(true);

    System.out.printf("WAL is disabled - %s \n", writeOptions.disableWAL());

    try {
        db = RocksDB.open(options, dbDirectory.getPath());
    } catch (RocksDBException e) {
        e.printStackTrace();
    }

}
 
Example #8
Source File: RocksDbConfiguration.java    From teku with Apache License 2.0 4 votes vote down vote up
public CompressionType getCompressionType() {
  return compressionType;
}
 
Example #9
Source File: RocksDbConfiguration.java    From teku with Apache License 2.0 4 votes vote down vote up
public CompressionType getBottomMostCompressionType() {
  return bottomMostCompressionType;
}
 
Example #10
Source File: RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java    From kcache with Apache License 2.0 4 votes vote down vote up
@Override
public CompressionType compressionType() {
    return columnFamilyOptions.compressionType();
}
 
Example #11
Source File: RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java    From kcache with Apache License 2.0 4 votes vote down vote up
@Override
public Options setCompressionPerLevel(final List<CompressionType> compressionLevels) {
    columnFamilyOptions.setCompressionPerLevel(compressionLevels);
    return this;
}
 
Example #12
Source File: RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java    From kcache with Apache License 2.0 4 votes vote down vote up
@Override
public List<CompressionType> compressionPerLevel() {
    return columnFamilyOptions.compressionPerLevel();
}
 
Example #13
Source File: RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java    From kcache with Apache License 2.0 4 votes vote down vote up
@Override
public Options setCompressionType(final CompressionType compressionType) {
    columnFamilyOptions.setCompressionType(compressionType);
    return this;
}
 
Example #14
Source File: RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java    From kcache with Apache License 2.0 4 votes vote down vote up
@Override
public Options setBottommostCompressionType(final CompressionType bottommostCompressionType) {
    columnFamilyOptions.setBottommostCompressionType(bottommostCompressionType);
    return this;
}
 
Example #15
Source File: RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.java    From kcache with Apache License 2.0 4 votes vote down vote up
@Override
public CompressionType bottommostCompressionType() {
    return columnFamilyOptions.bottommostCompressionType();
}