org.rocksdb.NativeLibraryLoader Java Examples

The following examples show how to use org.rocksdb.NativeLibraryLoader. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RocksDBPerformanceTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 
Example #2
Source File: RocksDBPerformanceTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 
Example #3
Source File: RocksDBPerformanceTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 
Example #4
Source File: RocksDBStateBackend.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private void ensureRocksDBIsLoaded(String tempDirectory) throws IOException {
	synchronized (RocksDBStateBackend.class) {
		if (!rocksDbInitialized) {

			final File tempDirParent = new File(tempDirectory).getAbsoluteFile();
			LOG.info("Attempting to load RocksDB native library and store it under '{}'", tempDirParent);

			Throwable lastException = null;
			for (int attempt = 1; attempt <= ROCKSDB_LIB_LOADING_ATTEMPTS; attempt++) {
				try {
					// when multiple instances of this class and RocksDB exist in different
					// class loaders, then we can see the following exception:
					// "java.lang.UnsatisfiedLinkError: Native Library /path/to/temp/dir/librocksdbjni-linux64.so
					// already loaded in another class loader"

					// to avoid that, we need to add a random element to the library file path
					// (I know, seems like an unnecessary hack, since the JVM obviously can handle multiple
					//  instances of the same JNI library being loaded in different class loaders, but
					//  apparently not when coming from the same file path, so there we go)

					final File rocksLibFolder = new File(tempDirParent, "rocksdb-lib-" + new AbstractID());

					// make sure the temp path exists
					LOG.debug("Attempting to create RocksDB native library folder {}", rocksLibFolder);
					// noinspection ResultOfMethodCallIgnored
					rocksLibFolder.mkdirs();

					// explicitly load the JNI dependency if it has not been loaded before
					NativeLibraryLoader.getInstance().loadLibrary(rocksLibFolder.getAbsolutePath());

					// this initialization here should validate that the loading succeeded
					RocksDB.loadLibrary();

					// seems to have worked
					LOG.info("Successfully loaded RocksDB native library");
					rocksDbInitialized = true;
					return;
				}
				catch (Throwable t) {
					lastException = t;
					LOG.debug("RocksDB JNI library loading attempt {} failed", attempt, t);

					// try to force RocksDB to attempt reloading the library
					try {
						resetRocksDBLoadedFlag();
					} catch (Throwable tt) {
						LOG.debug("Failed to reset 'initialized' flag in RocksDB native code loader", tt);
					}
				}
			}

			throw new IOException("Could not load the native RocksDB library", lastException);
		}
	}
}
 
Example #5
Source File: RocksDBStateBackend.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
static void resetRocksDBLoadedFlag() throws Exception {
	final Field initField = org.rocksdb.NativeLibraryLoader.class.getDeclaredField("initialized");
	initField.setAccessible(true);
	initField.setBoolean(null, false);
}
 
Example #6
Source File: RocksDBListStatePerformanceTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 
Example #7
Source File: RocksDBStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up
private void ensureRocksDBIsLoaded(String tempDirectory) throws IOException {
	synchronized (RocksDBStateBackend.class) {
		if (!rocksDbInitialized) {

			final File tempDirParent = new File(tempDirectory).getAbsoluteFile();
			LOG.info("Attempting to load RocksDB native library and store it under '{}'", tempDirParent);

			Throwable lastException = null;
			for (int attempt = 1; attempt <= ROCKSDB_LIB_LOADING_ATTEMPTS; attempt++) {
				try {
					// when multiple instances of this class and RocksDB exist in different
					// class loaders, then we can see the following exception:
					// "java.lang.UnsatisfiedLinkError: Native Library /path/to/temp/dir/librocksdbjni-linux64.so
					// already loaded in another class loader"

					// to avoid that, we need to add a random element to the library file path
					// (I know, seems like an unnecessary hack, since the JVM obviously can handle multiple
					//  instances of the same JNI library being loaded in different class loaders, but
					//  apparently not when coming from the same file path, so there we go)

					final File rocksLibFolder = new File(tempDirParent, "rocksdb-lib-" + new AbstractID());

					// make sure the temp path exists
					LOG.debug("Attempting to create RocksDB native library folder {}", rocksLibFolder);
					// noinspection ResultOfMethodCallIgnored
					rocksLibFolder.mkdirs();

					// explicitly load the JNI dependency if it has not been loaded before
					NativeLibraryLoader.getInstance().loadLibrary(rocksLibFolder.getAbsolutePath());

					// this initialization here should validate that the loading succeeded
					RocksDB.loadLibrary();

					// seems to have worked
					LOG.info("Successfully loaded RocksDB native library");
					rocksDbInitialized = true;
					return;
				}
				catch (Throwable t) {
					lastException = t;
					LOG.debug("RocksDB JNI library loading attempt {} failed", attempt, t);

					// try to force RocksDB to attempt reloading the library
					try {
						resetRocksDBLoadedFlag();
					} catch (Throwable tt) {
						LOG.debug("Failed to reset 'initialized' flag in RocksDB native code loader", tt);
					}
				}
			}

			throw new IOException("Could not load the native RocksDB library", lastException);
		}
	}
}
 
Example #8
Source File: RocksDBStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
static void resetRocksDBLoadedFlag() throws Exception {
	final Field initField = org.rocksdb.NativeLibraryLoader.class.getDeclaredField("initialized");
	initField.setAccessible(true);
	initField.setBoolean(null, false);
}
 
Example #9
Source File: RocksDBListStatePerformanceTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 
Example #10
Source File: RocksDBStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
static void resetRocksDBLoadedFlag() throws Exception {
	final Field initField = org.rocksdb.NativeLibraryLoader.class.getDeclaredField("initialized");
	initField.setAccessible(true);
	initField.setBoolean(null, false);
}
 
Example #11
Source File: RocksDBOperationsUtilsTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void loadRocksLibrary() throws Exception {
	NativeLibraryLoader.getInstance().loadLibrary(TMP_DIR.newFolder().getAbsolutePath());
}
 
Example #12
Source File: RocksDBListStatePerformanceTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 
Example #13
Source File: RocksDBMemoryControllerUtilsTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Before
public void ensureRocksDbNativeLibraryLoaded() throws IOException {
	NativeLibraryLoader.getInstance().loadLibrary(temporaryFolder.newFolder().getAbsolutePath());
}
 
Example #14
Source File: RocksDBResourceContainerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void ensureRocksDbNativeLibraryLoaded() throws IOException {
	NativeLibraryLoader.getInstance().loadLibrary(TMP_FOLDER.newFolder().getAbsolutePath());
}