Java Code Examples for org.rocksdb.NativeLibraryLoader

The following examples show how to use org.rocksdb.NativeLibraryLoader. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: RocksDBPerformanceTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 
Example 2
Source Project: flink   Source File: RocksDBPerformanceTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 
Example 3
Source Project: flink   Source File: RocksDBPerformanceTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void init() throws IOException {
	rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	options = new Options()
			.setCompactionStyle(CompactionStyle.LEVEL)
			.setLevelCompactionDynamicLevelBytes(true)
			.setIncreaseParallelism(4)
			.setUseFsync(false)
			.setMaxOpenFiles(-1)
			.setCreateIfMissing(true)
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

	writeOptions = new WriteOptions()
			.setSync(false)
			.setDisableWAL(true);
}
 
Example 4
Source Project: Flink-CEPplus   Source File: RocksDBStateBackend.java    License: Apache License 2.0 4 votes vote down vote up
private void ensureRocksDBIsLoaded(String tempDirectory) throws IOException {
	synchronized (RocksDBStateBackend.class) {
		if (!rocksDbInitialized) {

			final File tempDirParent = new File(tempDirectory).getAbsoluteFile();
			LOG.info("Attempting to load RocksDB native library and store it under '{}'", tempDirParent);

			Throwable lastException = null;
			for (int attempt = 1; attempt <= ROCKSDB_LIB_LOADING_ATTEMPTS; attempt++) {
				try {
					// when multiple instances of this class and RocksDB exist in different
					// class loaders, then we can see the following exception:
					// "java.lang.UnsatisfiedLinkError: Native Library /path/to/temp/dir/librocksdbjni-linux64.so
					// already loaded in another class loader"

					// to avoid that, we need to add a random element to the library file path
					// (I know, seems like an unnecessary hack, since the JVM obviously can handle multiple
					//  instances of the same JNI library being loaded in different class loaders, but
					//  apparently not when coming from the same file path, so there we go)

					final File rocksLibFolder = new File(tempDirParent, "rocksdb-lib-" + new AbstractID());

					// make sure the temp path exists
					LOG.debug("Attempting to create RocksDB native library folder {}", rocksLibFolder);
					// noinspection ResultOfMethodCallIgnored
					rocksLibFolder.mkdirs();

					// explicitly load the JNI dependency if it has not been loaded before
					NativeLibraryLoader.getInstance().loadLibrary(rocksLibFolder.getAbsolutePath());

					// this initialization here should validate that the loading succeeded
					RocksDB.loadLibrary();

					// seems to have worked
					LOG.info("Successfully loaded RocksDB native library");
					rocksDbInitialized = true;
					return;
				}
				catch (Throwable t) {
					lastException = t;
					LOG.debug("RocksDB JNI library loading attempt {} failed", attempt, t);

					// try to force RocksDB to attempt reloading the library
					try {
						resetRocksDBLoadedFlag();
					} catch (Throwable tt) {
						LOG.debug("Failed to reset 'initialized' flag in RocksDB native code loader", tt);
					}
				}
			}

			throw new IOException("Could not load the native RocksDB library", lastException);
		}
	}
}
 
Example 5
Source Project: Flink-CEPplus   Source File: RocksDBStateBackend.java    License: Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
static void resetRocksDBLoadedFlag() throws Exception {
	final Field initField = org.rocksdb.NativeLibraryLoader.class.getDeclaredField("initialized");
	initField.setAccessible(true);
	initField.setBoolean(null, false);
}
 
Example 6
@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 
Example 7
Source Project: flink   Source File: RocksDBStateBackend.java    License: Apache License 2.0 4 votes vote down vote up
private void ensureRocksDBIsLoaded(String tempDirectory) throws IOException {
	synchronized (RocksDBStateBackend.class) {
		if (!rocksDbInitialized) {

			final File tempDirParent = new File(tempDirectory).getAbsoluteFile();
			LOG.info("Attempting to load RocksDB native library and store it under '{}'", tempDirParent);

			Throwable lastException = null;
			for (int attempt = 1; attempt <= ROCKSDB_LIB_LOADING_ATTEMPTS; attempt++) {
				try {
					// when multiple instances of this class and RocksDB exist in different
					// class loaders, then we can see the following exception:
					// "java.lang.UnsatisfiedLinkError: Native Library /path/to/temp/dir/librocksdbjni-linux64.so
					// already loaded in another class loader"

					// to avoid that, we need to add a random element to the library file path
					// (I know, seems like an unnecessary hack, since the JVM obviously can handle multiple
					//  instances of the same JNI library being loaded in different class loaders, but
					//  apparently not when coming from the same file path, so there we go)

					final File rocksLibFolder = new File(tempDirParent, "rocksdb-lib-" + new AbstractID());

					// make sure the temp path exists
					LOG.debug("Attempting to create RocksDB native library folder {}", rocksLibFolder);
					// noinspection ResultOfMethodCallIgnored
					rocksLibFolder.mkdirs();

					// explicitly load the JNI dependency if it has not been loaded before
					NativeLibraryLoader.getInstance().loadLibrary(rocksLibFolder.getAbsolutePath());

					// this initialization here should validate that the loading succeeded
					RocksDB.loadLibrary();

					// seems to have worked
					LOG.info("Successfully loaded RocksDB native library");
					rocksDbInitialized = true;
					return;
				}
				catch (Throwable t) {
					lastException = t;
					LOG.debug("RocksDB JNI library loading attempt {} failed", attempt, t);

					// try to force RocksDB to attempt reloading the library
					try {
						resetRocksDBLoadedFlag();
					} catch (Throwable tt) {
						LOG.debug("Failed to reset 'initialized' flag in RocksDB native code loader", tt);
					}
				}
			}

			throw new IOException("Could not load the native RocksDB library", lastException);
		}
	}
}
 
Example 8
Source Project: flink   Source File: RocksDBStateBackend.java    License: Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
static void resetRocksDBLoadedFlag() throws Exception {
	final Field initField = org.rocksdb.NativeLibraryLoader.class.getDeclaredField("initialized");
	initField.setAccessible(true);
	initField.setBoolean(null, false);
}
 
Example 9
Source Project: flink   Source File: RocksDBListStatePerformanceTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 
Example 10
Source Project: flink   Source File: RocksDBStateBackend.java    License: Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
static void resetRocksDBLoadedFlag() throws Exception {
	final Field initField = org.rocksdb.NativeLibraryLoader.class.getDeclaredField("initialized");
	initField.setAccessible(true);
	initField.setBoolean(null, false);
}
 
Example 11
Source Project: flink   Source File: RocksDBOperationsUtilsTest.java    License: Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void loadRocksLibrary() throws Exception {
	NativeLibraryLoader.getInstance().loadLibrary(TMP_DIR.newFolder().getAbsolutePath());
}
 
Example 12
Source Project: flink   Source File: RocksDBListStatePerformanceTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbListStateAPIs() throws Exception {
	final File rocksDir = tmp.newFolder();

	// ensure the RocksDB library is loaded to a distinct location each retry
	NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());

	final String key1 = "key1";
	final String key2 = "key2";
	final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";

	final byte[] keyBytes1 = key1.getBytes(StandardCharsets.UTF_8);
	final byte[] keyBytes2 = key2.getBytes(StandardCharsets.UTF_8);
	final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);

	// The number of values added to ListState. Can be changed for benchmarking
	final int num = 10;

	try (
		final Options options = new Options()
				.setCompactionStyle(CompactionStyle.LEVEL)
				.setLevelCompactionDynamicLevelBytes(true)
				.setIncreaseParallelism(4)
				.setUseFsync(false)
				.setMaxOpenFiles(-1)
				.setCreateIfMissing(true)
				.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);

		final WriteOptions writeOptions = new WriteOptions()
				.setSync(false)
				.setDisableWAL(true);

		final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {

		// ----- add() API -----
		log.info("begin add");

		final long beginInsert1 = System.nanoTime();
		for (int i = 0; i < num; i++) {
			rocksDB.merge(writeOptions, keyBytes1, valueBytes);
		}
		final long endInsert1 = System.nanoTime();

		log.info("end add - duration: {} ns", (endInsert1 - beginInsert1));

		// ----- update() API -----

		List<byte[]> list = new ArrayList<>(num);
		for (int i = 0; i < num; i++) {
			list.add(valueBytes);
		}
		byte[] premerged = merge(list);

		log.info("begin update");

		final long beginInsert2 = System.nanoTime();
		rocksDB.merge(writeOptions, keyBytes2, premerged);
		final long endInsert2 = System.nanoTime();

		log.info("end update - duration: {} ns", (endInsert2 - beginInsert2));
	}
}
 
Example 13
Source Project: flink   Source File: RocksDBMemoryControllerUtilsTest.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void ensureRocksDbNativeLibraryLoaded() throws IOException {
	NativeLibraryLoader.getInstance().loadLibrary(temporaryFolder.newFolder().getAbsolutePath());
}
 
Example 14
Source Project: flink   Source File: RocksDBResourceContainerTest.java    License: Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void ensureRocksDbNativeLibraryLoaded() throws IOException {
	NativeLibraryLoader.getInstance().loadLibrary(TMP_FOLDER.newFolder().getAbsolutePath());
}