com.sleepycat.je.EnvironmentConfig Java Examples

The following examples show how to use com.sleepycat.je.EnvironmentConfig. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: qpid-broker-j   Author: apache   File: StandardEnvironmentFacadeFactoryTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateEnvironmentFacade()
{
    when(_parent.getName()).thenReturn(getTestName());
    when(_parent.getContextKeys(any(boolean.class))).thenReturn(Collections.singleton(EnvironmentConfig.ENV_IS_TRANSACTIONAL));
    when(_parent.getContextValue(String.class, EnvironmentConfig.ENV_IS_TRANSACTIONAL)).thenReturn("false");

    StandardEnvironmentFacadeFactory factory = new StandardEnvironmentFacadeFactory();
    EnvironmentFacade facade = factory.createEnvironmentFacade(_parent);
    try
    {
        facade.beginTransaction(null);
        fail("Context variables were not picked up on environment creation");
    }
    catch(UnsupportedOperationException e)
    {
        //pass
    }
}
 
Example #2
Source Project: qpid-broker-j   Author: apache   File: StandardEnvironmentFacadeTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOverrideJeParameter() throws Exception
{
    // verify that transactions can be created by default
    EnvironmentFacade ef = createEnvironmentFacade();
    Transaction t = ef.beginTransaction(null);
    t.commit();
    ef.close();

    // customize the environment to be non-transactional
    ef = createEnvironmentFacade(Collections.singletonMap(EnvironmentConfig.ENV_IS_TRANSACTIONAL, "false"));
    try
    {
        ef.beginTransaction(null);
        fail("Overridden settings were not picked up on environment creation");
    }
    catch(UnsupportedOperationException e)
    {
        // pass
    }
    ef.close();
}
 
Example #3
Source Project: bboxdb   Author: jnidzwetzki   File: TestKDTreeSplit.java    License: Apache License 2.0 6 votes vote down vote up
public TestKDTreeSplit(final File tmpDir, final List<Pair<String, String>> filesAndFormats,
		final List<Integer> experimentSize) {

	this.filesAndFormats = filesAndFormats;
	this.experimentSize = experimentSize;

	this.elements = new HashMap<>();
	this.elementCounter = new HashMap<>();
	this.boxDimension = new HashMap<>();

	// Setup database dir
	tmpDir.mkdirs();
	FileUtil.deleteDirOnExit(tmpDir.toPath());

	final EnvironmentConfig envConfig = new EnvironmentConfig();
	envConfig.setTransactional(false);
	envConfig.setAllowCreate(true);
    envConfig.setSharedCache(true);
	dbEnv = new Environment(tmpDir, envConfig);

	dbConfig = new DatabaseConfig();
	dbConfig.setTransactional(false);
	dbConfig.setAllowCreate(true);
}
 
Example #4
Source Project: bboxdb   Author: jnidzwetzki   File: FileLineIndex.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Open the Berkeley DB
 * @throws IOException
 */
protected void openDatabase() throws IOException {
	final EnvironmentConfig envConfig = new EnvironmentConfig();
	envConfig.setTransactional(false);
	envConfig.setAllowCreate(true);
    envConfig.setSharedCache(true);

    tmpDatabaseDir = Files.createTempDirectory(null);
	dbEnv = new Environment(tmpDatabaseDir.toFile(), envConfig);

	logger.info("Database dir is {}", tmpDatabaseDir);

	// Delete database on exit
	FileUtil.deleteDirOnExit(tmpDatabaseDir);

	final DatabaseConfig dbConfig = new DatabaseConfig();
	dbConfig.setTransactional(false);
	dbConfig.setAllowCreate(true);
	dbConfig.setDeferredWrite(true);

	database = dbEnv.openDatabase(null, "lines", dbConfig);
}
 
Example #5
Source Project: tddl5   Author: loye168   File: JE_Repository.java    License: Apache License 2.0 6 votes vote down vote up
public void commonConfig(EnvironmentConfig envConfig) {
    System.setProperty("JEMonitor", "true");
    envConfig.setConfigParam(EnvironmentConfig.NODE_MAX_ENTRIES, "256");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_EVICT_BYTES, (1024 * 1024 * 2) + "");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_NODES_PER_SCAN, "10");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_LRU_ONLY, "false");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_FORCED_YIELD, "true");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_CORE_THREADS, Runtime.getRuntime().availableProcessors()
                                                                     + "");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_MAX_THREADS, Runtime.getRuntime().availableProcessors() + "");
    envConfig.setConfigParam(EnvironmentConfig.CHECKPOINTER_BYTES_INTERVAL, 1024 * 1024 * 200 + "");
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_LAZY_MIGRATION, "true");
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_THREADS, Runtime.getRuntime().availableProcessors() + "");
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_LOOK_AHEAD_CACHE_SIZE, 1024 * 8 + "");
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_READ_SIZE, 1024 * 1024 + "");
    envConfig.setConfigParam(EnvironmentConfig.LOG_FILE_MAX, 1024 * 1024 * 200 + "");
    envConfig.setConfigParam(EnvironmentConfig.LOG_FILE_CACHE_SIZE, "1024");
    envConfig.setConfigParam(EnvironmentConfig.LOG_USE_WRITE_QUEUE, "true");
    envConfig.setConfigParam(EnvironmentConfig.LOG_WRITE_QUEUE_SIZE, 1024 * 1024 * 2 + "");
    // envConfig.setConfigParam(EnvironmentConfig.HALT_ON_COMMIT_AFTER_CHECKSUMEXCEPTION,
    // "true");
    envConfig.setConfigParam(EnvironmentConfig.LOG_ITERATOR_READ_SIZE, 1024 * 8 + "");
    envConfig.setConfigParam(EnvironmentConfig.LOCK_TIMEOUT, 2000 + "\tMILLISECONDS");
    envConfig.setConfigParam(EnvironmentConfig.ENV_RECOVERY_FORCE_CHECKPOINT, "true");
}
 
Example #6
Source Project: codes-scratch-crawler   Author: classtag   File: WebGraph.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 构造函数
 */
public WebGraph(String dbDir) throws DatabaseException {
  File envDir = new File(dbDir);
  EnvironmentConfig envConfig = new EnvironmentConfig();
  envConfig.setTransactional(false);
  envConfig.setAllowCreate(true);
  Environment env = new Environment(envDir, envConfig);

  StoreConfig storeConfig = new StoreConfig();
  storeConfig.setAllowCreate(true);
  storeConfig.setTransactional(false);

  store = new EntityStore(env, "classDb", storeConfig);
  outLinkIndex = store.getPrimaryIndex(String.class, Link.class);
  inLinkIndex = store.getSecondaryIndex(outLinkIndex, String.class, "toURL");
}
 
Example #7
Source Project: codes-scratch-crawler   Author: classtag   File: AbstractFrontier.java    License: Apache License 2.0 6 votes vote down vote up
public AbstractFrontier(String homeDirectory) {
  EnvironmentConfig envConfig = new EnvironmentConfig();
  envConfig.setTransactional(true);
  envConfig.setAllowCreate(true);
  env = new Environment(new File(homeDirectory), envConfig);
  // 设置databaseconfig
  DatabaseConfig dbConfig = new DatabaseConfig();
  dbConfig.setTransactional(true);
  dbConfig.setAllowCreate(true);
  // 打开
  catalogdatabase = env.openDatabase(null, CLASS_CATALOG, dbConfig);
  javaCatalog = new StoredClassCatalog(catalogdatabase);
  // 设置databaseconfig
  DatabaseConfig dbConfig0 = new DatabaseConfig();
  dbConfig0.setTransactional(true);
  dbConfig0.setAllowCreate(true);
  database = env.openDatabase(null, "URL", dbConfig0);
}
 
Example #8
Source Project: hypergraphdb   Author: hypergraphdb   File: BJEConfig.java    License: Apache License 2.0 6 votes vote down vote up
private void resetDefaults(boolean readOnly)
{
	envConfig.setReadOnly(readOnly);
	dbConfig.setReadOnly(readOnly);

	envConfig.setAllowCreate(!readOnly);
	dbConfig.setAllowCreate(!readOnly);

	// envConfig.setCacheSize(DEFAULT_STORE_CACHE);
	envConfig.setCachePercent(30);

	envConfig.setConfigParam(EnvironmentConfig.LOG_FILE_MAX,
			Long.toString(10000000 * 10l));
	envConfig.setConfigParam(
			EnvironmentConfig.CLEANER_LOOK_AHEAD_CACHE_SIZE,
			Long.toString(1024 * 1024));
	envConfig.setConfigParam(EnvironmentConfig.CLEANER_READ_SIZE,
			Long.toString(1024 * 1024));
}
 
Example #9
Source Project: tddl   Author: cpsing   File: JE_Repository.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void init() {

    EnvironmentConfig envConfig = new EnvironmentConfig();
    commonConfig(envConfig);
    envConfig.setCachePercent(config.getCachePercent());
    envConfig.setAllowCreate(true);
    if (config.isTransactional()) {
        envConfig.setCachePercent(config.getCachePercent());
        envConfig.setTransactional(config.isTransactional());
        envConfig.setTxnTimeout(config.getTxnTimeout(), TimeUnit.SECONDS);
        this.durability = config.isCommitSync() ? Durability.COMMIT_SYNC : Durability.COMMIT_NO_SYNC;
        envConfig.setDurability(this.durability);
    }
    File repo_dir = new File(config.getRepoDir());
    if (!repo_dir.exists()) {
        repo_dir.mkdirs();
    }
    this.env = new Environment(repo_dir, envConfig);
    cef = new CommandHandlerFactoryBDBImpl();
    cursorFactoryBDBImp = new CursorFactoryBDBImp();

}
 
Example #10
Source Project: tddl   Author: cpsing   File: JE_Repository.java    License: Apache License 2.0 6 votes vote down vote up
public void commonConfig(EnvironmentConfig envConfig) {
    System.setProperty("JEMonitor", "true");
    envConfig.setConfigParam(EnvironmentConfig.NODE_MAX_ENTRIES, "256");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_EVICT_BYTES, (1024 * 1024 * 2) + "");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_NODES_PER_SCAN, "10");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_LRU_ONLY, "false");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_FORCED_YIELD, "true");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_CORE_THREADS, Runtime.getRuntime().availableProcessors()
                                                                     + "");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_MAX_THREADS, Runtime.getRuntime().availableProcessors() + "");
    envConfig.setConfigParam(EnvironmentConfig.CHECKPOINTER_BYTES_INTERVAL, 1024 * 1024 * 200 + "");
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_LAZY_MIGRATION, "true");
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_THREADS, Runtime.getRuntime().availableProcessors() + "");
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_LOOK_AHEAD_CACHE_SIZE, 1024 * 8 + "");
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_READ_SIZE, 1024 * 1024 + "");
    envConfig.setConfigParam(EnvironmentConfig.LOG_FILE_MAX, 1024 * 1024 * 200 + "");
    envConfig.setConfigParam(EnvironmentConfig.LOG_FILE_CACHE_SIZE, "1024");
    envConfig.setConfigParam(EnvironmentConfig.LOG_USE_WRITE_QUEUE, "true");
    envConfig.setConfigParam(EnvironmentConfig.LOG_WRITE_QUEUE_SIZE, 1024 * 1024 * 2 + "");
    // envConfig.setConfigParam(EnvironmentConfig.HALT_ON_COMMIT_AFTER_CHECKSUMEXCEPTION,
    // "true");
    envConfig.setConfigParam(EnvironmentConfig.LOG_ITERATOR_READ_SIZE, 1024 * 8 + "");
    envConfig.setConfigParam(EnvironmentConfig.LOCK_TIMEOUT, 2000 + "\tMILLISECONDS");
    envConfig.setConfigParam(EnvironmentConfig.ENV_RECOVERY_FORCE_CHECKPOINT, "true");
}
 
Example #11
Source Project: Rel   Author: DaveVoorhis   File: ClassCatalog.java    License: Apache License 2.0 6 votes vote down vote up
ClassCatalog(String directory, EnvironmentConfig environmentConfig, DatabaseConfig dbConfig) {
	// This should be main database directory
	dirClassLoader = new DirClassLoader(directory);
	
    // Open the environment in subdirectory of the above
	String classesDir = directory + java.io.File.separator + "classes";
	RelDatabase.mkdir(classesDir);
    environment = new Environment(new File(classesDir), environmentConfig);
    
    // Open the class catalog db. This is used to optimize class serialization.
    classCatalogDb = environment.openDatabase(null, "_ClassCatalog", dbConfig);

    // Create our class catalog
    classCatalog = new StoredClassCatalog(classCatalogDb);
    
    // Need a serial binding for metadata
    relvarMetadataBinding = new SerialBinding<RelvarMetadata>(classCatalog, RelvarMetadata.class);
    
    // Need serial binding for data
    tupleBinding = new SerialBinding<ValueTuple>(classCatalog, ValueTuple.class) {
    	public ClassLoader getClassLoader() {
    		return dirClassLoader;
    	}
    };   	
}
 
Example #12
Source Project: jstarcraft-core   Author: HongZhaoHua   File: Migrator.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 构建新数据库环境
 * 
 * @param databaseConfiguration
 */
public void buildNewEnvironment(BerkeleyAccessor accessorFactory) {
    // 删除准备迁移的仓储
    final EnvironmentConfig environmentConfiguration = new EnvironmentConfig(environmentProperties);
    final Environment environment = new Environment(this.newDatabaseDirectory, environmentConfiguration);

    for (String databaseName : environment.getDatabaseNames()) {
        // 检测数据库名称是否符合EntityStore的名称规范(persist#STORE_NAME#ENTITY_CLASS)
        if (databaseName.startsWith(PERSIST_PREFIX)) {
            final String[] databasePartNames = databaseName.split("#");
            logger.debug("扫描数据库[{}]", databaseName);
            if (this.migrateStroreNames.contains(databasePartNames[1])) {
                environment.removeDatabase(null, databaseName);
                logger.debug("删除数据库[{}]", databaseName);
            }
        }
    }

    environment.close();

    // 让环境自动构建迁移的实体数据表
    // final ClassPathXmlApplicationContext applicationContext = new
    // ClassPathXmlApplicationContext(new String[] { databaseConfiguration });
    // BerkeleyAccessor accessorFactory =
    // applicationContext.getBean(BerkeleyAccessor.class);
    // applicationContext.close();
    // while (accessorFactory.getState() != BerkeleyState.STOPPED) {
    // Thread.yield();
    // }
    accessorFactory.start();
    accessorFactory.stop();
}
 
Example #13
Source Project: qpid-broker-j   Author: apache   File: BDBUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static Map<String, String> getEnvironmentConfigurationParameters(ConfiguredObject<?> object)
{
    Map<String, String> parameters = getContextSettingsWithNameMatchingRegExpPattern(object, NON_REP_JE_PARAM_PATTERN);
    if (!parameters.containsKey(EnvironmentConfig.MAX_MEMORY) && !parameters.containsKey(EnvironmentConfig.MAX_MEMORY_PERCENT))
    {
        parameters.put(EnvironmentConfig.MAX_MEMORY, String.valueOf(BDBVirtualHost.BDB_MIN_CACHE_SIZE));
    }
    return Collections.unmodifiableMap(parameters);
}
 
Example #14
Source Project: qpid-broker-j   Author: apache   File: AbstractUpgradeTestCase.java    License: Apache License 2.0 5 votes vote down vote up
protected Environment createEnvironment(File storeLocation)
{
    EnvironmentConfig envConfig = new EnvironmentConfig();
    envConfig.setAllowCreate(true);
    envConfig.setTransactional(true);
    envConfig.setConfigParam("je.lock.nLockTables", "7");
    envConfig.setReadOnly(false);
    envConfig.setSharedCache(false);
    envConfig.setCacheSize(0);
    return new Environment(storeLocation, envConfig);
}
 
Example #15
Source Project: qpid-broker-j   Author: apache   File: BDBPreferenceStoreTest.java    License: Apache License 2.0 5 votes vote down vote up
private void populateTestData(final List<PreferenceRecord> records, final String modelVersion)
{
    EnvironmentConfig envConfig = new EnvironmentConfig();
    envConfig.setAllowCreate(true);
    envConfig.setTransactional(false);
    try (Environment environment = new Environment(_storeFile, envConfig))
    {
        DatabaseConfig dbConfig = new DatabaseConfig();
        dbConfig.setAllowCreate(true);
        try (Database versionDb = environment.openDatabase(null, "USER_PREFERENCES_VERSION", dbConfig);
             Database preferencesDb = environment.openDatabase(null, "USER_PREFERENCES", dbConfig))
        {
            DatabaseEntry key = new DatabaseEntry();
            DatabaseEntry value = new DatabaseEntry();
            UUIDTupleBinding keyBinding = UUIDTupleBinding.getInstance();
            MapBinding valueBinding = MapBinding.getInstance();
            for (PreferenceRecord record : records)
            {
                keyBinding.objectToEntry(record.getId(), key);
                valueBinding.objectToEntry(record.getAttributes(), value);
                preferencesDb.put(null, key, value);
            }

            ByteBinding.byteToEntry((byte) 0, value);
            StringBinding.stringToEntry(modelVersion, key);
            versionDb.put(null, key, value);
        }
    }
}
 
Example #16
Source Project: SPADE   Author: ashish-gehani   File: BerkeleyDBManager.java    License: GNU General Public License v3.0 5 votes vote down vote up
/**
 * Creates BerkeleyDB environment object
 * 
 * @param path environment path which must already exist
 * @return BerkeleyDB environment object or error
 */
private synchronized Result<Environment> createBerkeleyDBEnvironmentObject(String path){
	try{
		EnvironmentConfig envConfig = new EnvironmentConfig();
		envConfig.setAllowCreate(true);
		Environment environment = new Environment(new File(path), envConfig);
		return Result.successful(environment);
	}catch(Exception e){
		return Result.failed("Failed to create BerkeleyDB environment object", e, null);
	}
}
 
Example #17
Source Project: SPADE   Author: ashish-gehani   File: CompressedStorage.java    License: GNU General Public License v3.0 5 votes vote down vote up
/**
 * This method is invoked by the kernel to initialize the storage.
 *
 * @param arguments The arguments with which this storage is to be
 *                  initialized.
 * @return True if the storage was initialized successfully.
 */
public boolean initialize(String filePath) {
	//clock = System.currentTimeMillis();
	countUpdates = 0;
	annotationsTime = 0;
	scaffoldTime = 0;
	clockScaffold = 0;
	clockAnnotations = 0;
	scaffoldInMemory = new HashMap<Integer, Pair<SortedSet<Integer>, SortedSet<Integer>>>();
	edgesInMemory = 0;
	hashToID = new HashMap<String, Integer>();
	idToHash = new HashMap<Integer, String>();
	alreadyRenamed = new Vector<String>();
	compresser = new Deflater(Deflater.BEST_COMPRESSION);
	W=10;
	L=5;
	nextVertexID = 0;
	try {
		benchmarks = new PrintWriter("/Users/melanie/Documents/benchmarks/compression_time_berkeleyDB.txt", "UTF-8");
		// Open the environment. Create it if it does not already exist.
		EnvironmentConfig envConfig = new EnvironmentConfig();
		envConfig.setAllowCreate(true);
		DatabaseEnvironment1 = new Environment(new File(filePath + "/scaffold"), 
				envConfig);
		DatabaseEnvironment2 = new Environment(new File(filePath + "/annotations"), 
				envConfig);
		// Open the databases. Create it if it does not already exist.
		DatabaseConfig DatabaseConfig1 = new DatabaseConfig();
		DatabaseConfig1.setAllowCreate(true);
		scaffoldDatabase = DatabaseEnvironment1.openDatabase(null, "spade_scaffold", DatabaseConfig1); 
		annotationsDatabase = DatabaseEnvironment2.openDatabase(null, "spade_annotations", DatabaseConfig1); 

		return true;

	} catch (Exception ex) {
		// Exception handling goes here
		logger.log(Level.SEVERE, "Compressed Storage Initialized not successful!", ex);
		return false;
	}
}
 
Example #18
Source Project: bboxdb   Author: jnidzwetzki   File: BDBTupleStore.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open() throws Exception {
	final EnvironmentConfig envConfig = new EnvironmentConfig();
	envConfig.setTransactional(USE_TRANSACTIONS);
	envConfig.setAllowCreate(true);

	environment = new Environment(dir, envConfig);

	Transaction txn = null;
	if(USE_TRANSACTIONS) {
		txn = environment.beginTransaction(null, null);
	}
	
	final DatabaseConfig dbConfig = new DatabaseConfig();
	dbConfig.setTransactional(USE_TRANSACTIONS);
	dbConfig.setAllowCreate(true);
	//dbConfig.setSortedDuplicates(true);
	dbConfig.setDeferredWrite(true);
	//dbConfig.setKeyPrefixing(true);
	//dbConfig.setNodeMaxEntries(128);

	database = environment.openDatabase(txn, "test", dbConfig);

	if(txn != null) {
		txn.commit();
	}	
}
 
Example #19
Source Project: bboxdb   Author: jnidzwetzki   File: OSMBDBNodeStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Init a new BDB environment in the given folder
 * @param folder
 */
@SuppressWarnings("unused")
protected void initNewBDBEnvironment(final File folder, final EnvironmentConfig envConfig) {

	final Environment dbEnv = new Environment(folder, envConfig);

	Transaction txn = null;
	if(USE_TRANSACTIONS) {
		txn = dbEnv.beginTransaction(null, null);
	}
	
	final DatabaseConfig dbConfig = new DatabaseConfig();
	dbConfig.setTransactional(USE_TRANSACTIONS);
	dbConfig.setAllowCreate(true);
	dbConfig.setSortedDuplicates(true);
	dbConfig.setDeferredWrite(true);
	//dbConfig.setKeyPrefixing(true);
	//dbConfig.setNodeMaxEntries(128);

	final Database database = dbEnv.openDatabase(txn, "osm", dbConfig);

	if(txn != null) {
		txn.commit();
	}
	
	environments.add(dbEnv);
	databases.add(database);
}
 
Example #20
Source Project: tddl5   Author: loye168   File: JE_Repository.java    License: Apache License 2.0 5 votes vote down vote up
public void commonConfig(EnvironmentConfig envConfig, BDBConfig config) {
    System.setProperty("JEMonitor", "true");
    envConfig.setConfigParam(EnvironmentConfig.NODE_MAX_ENTRIES, "256");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_EVICT_BYTES, (1024 * 1024 * 2) + "");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_NODES_PER_SCAN, "10");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_LRU_ONLY, "false");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_FORCED_YIELD, "true");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_CORE_THREADS, Runtime.getRuntime().availableProcessors()
                                                                     + "");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_MAX_THREADS, Runtime.getRuntime().availableProcessors() + "");
    envConfig.setConfigParam(EnvironmentConfig.CHECKPOINTER_BYTES_INTERVAL, 1024 * 1024 * 200 + "");

    envConfig.setConfigParam(EnvironmentConfig.CLEANER_LOOK_AHEAD_CACHE_SIZE, 1024 * 8 + "");
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_READ_SIZE, 1024 * 1024 + "");
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_MAX_BATCH_FILES, 3 + "");
    envConfig.setConfigParam(EnvironmentConfig.LOG_FILE_MAX, 1024 * 1024 * 200 + "");
    envConfig.setConfigParam(EnvironmentConfig.LOG_FILE_CACHE_SIZE, "1024");
    envConfig.setConfigParam(EnvironmentConfig.LOG_USE_WRITE_QUEUE, "true");
    envConfig.setConfigParam(EnvironmentConfig.LOG_WRITE_QUEUE_SIZE, 1024 * 1024 * 2 + "");
    // envConfig.setConfigParam(EnvironmentConfig.HALT_ON_COMMIT_AFTER_CHECKSUMEXCEPTION,
    // "true");
    envConfig.setConfigParam(EnvironmentConfig.LOG_ITERATOR_READ_SIZE, 1024 * 8 + "");
    envConfig.setConfigParam(EnvironmentConfig.LOCK_TIMEOUT, 2000 + "\tMILLISECONDS");
    envConfig.setConfigParam(EnvironmentConfig.ENV_RECOVERY_FORCE_CHECKPOINT, "true");

    envConfig.setConfigParam(EnvironmentConfig.CLEANER_MIN_UTILIZATION, (config.getCleaner_min_utilization()));
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_LAZY_MIGRATION, config.getCleanerLazyMigration());
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_THREADS, config.getCleanerThreadCount() + "");
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_MAX_BATCH_FILES, config.getCleanerBatchFileCount() + "");
    envConfig.setConfigParam(EnvironmentConfig.ENV_RUN_CLEANER, config.getAutoClean() ? "true" : "false");

}
 
Example #21
Source Project: tddl   Author: cpsing   File: JE_Repository.java    License: Apache License 2.0 5 votes vote down vote up
public void commonConfig(EnvironmentConfig envConfig, BDBConfig config) {
    System.setProperty("JEMonitor", "true");
    envConfig.setConfigParam(EnvironmentConfig.NODE_MAX_ENTRIES, "256");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_EVICT_BYTES, (1024 * 1024 * 2) + "");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_NODES_PER_SCAN, "10");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_LRU_ONLY, "false");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_FORCED_YIELD, "true");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_CORE_THREADS, Runtime.getRuntime().availableProcessors()
                                                                     + "");
    envConfig.setConfigParam(EnvironmentConfig.EVICTOR_MAX_THREADS, Runtime.getRuntime().availableProcessors() + "");
    envConfig.setConfigParam(EnvironmentConfig.CHECKPOINTER_BYTES_INTERVAL, 1024 * 1024 * 200 + "");

    envConfig.setConfigParam(EnvironmentConfig.CLEANER_LOOK_AHEAD_CACHE_SIZE, 1024 * 8 + "");
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_READ_SIZE, 1024 * 1024 + "");
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_MAX_BATCH_FILES, 3 + "");
    envConfig.setConfigParam(EnvironmentConfig.LOG_FILE_MAX, 1024 * 1024 * 200 + "");
    envConfig.setConfigParam(EnvironmentConfig.LOG_FILE_CACHE_SIZE, "1024");
    envConfig.setConfigParam(EnvironmentConfig.LOG_USE_WRITE_QUEUE, "true");
    envConfig.setConfigParam(EnvironmentConfig.LOG_WRITE_QUEUE_SIZE, 1024 * 1024 * 2 + "");
    // envConfig.setConfigParam(EnvironmentConfig.HALT_ON_COMMIT_AFTER_CHECKSUMEXCEPTION,
    // "true");
    envConfig.setConfigParam(EnvironmentConfig.LOG_ITERATOR_READ_SIZE, 1024 * 8 + "");
    envConfig.setConfigParam(EnvironmentConfig.LOCK_TIMEOUT, 2000 + "\tMILLISECONDS");
    envConfig.setConfigParam(EnvironmentConfig.ENV_RECOVERY_FORCE_CHECKPOINT, "true");

    envConfig.setConfigParam(EnvironmentConfig.CLEANER_MIN_UTILIZATION, (config.getCleaner_min_utilization()));
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_LAZY_MIGRATION, config.getCleanerLazyMigration());
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_THREADS, config.getCleanerThreadCount() + "");
    envConfig.setConfigParam(EnvironmentConfig.CLEANER_MAX_BATCH_FILES, config.getCleanerBatchFileCount() + "");
    envConfig.setConfigParam(EnvironmentConfig.ENV_RUN_CLEANER, config.getAutoClean() ? "true" : "false");

}
 
Example #22
Source Project: WebCollector   Author: CrawlScript   File: BerkeleyDBReader.java    License: GNU General Public License v3.0 5 votes vote down vote up
public BerkeleyDBReader(String crawlPath) {
    this.crawlPath = crawlPath;
    File dir = new File(crawlPath);
    EnvironmentConfig environmentConfig = new EnvironmentConfig();
    environmentConfig.setAllowCreate(true);
    env = new Environment(dir, environmentConfig);
    crawldbDatabase = env.openDatabase(null, "crawldb", BerkeleyDBUtils.defaultDBConfig);
    cursor = crawldbDatabase.openCursor(null, CursorConfig.DEFAULT);
}
 
Example #23
Source Project: WebCollector   Author: CrawlScript   File: BerkeleyDBManager.java    License: GNU General Public License v3.0 5 votes vote down vote up
@Override
public void open() throws Exception {
    File dir = new File(crawlPath);
    if (!dir.exists()) {
        dir.mkdirs();
    }
    EnvironmentConfig environmentConfig = new EnvironmentConfig();
    environmentConfig.setAllowCreate(true);

    env = new Environment(dir, environmentConfig);
}
 
Example #24
Source Project: timbuctoo   Author: HuygensING   File: BdbPersistentEnvironmentCreator.java    License: GNU General Public License v3.0 5 votes vote down vote up
@JsonCreator
public BdbPersistentEnvironmentCreator(@JsonProperty("databaseLocation") String databaseLocation) {
  this.databaseLocation = databaseLocation;
  configuration = new EnvironmentConfig(new Properties());
  configuration.setTransactional(true);
  configuration.setDurability(Durability.COMMIT_NO_SYNC);
  configuration.setAllowCreate(true);
  configuration.setSharedCache(true);
  bdbBackupper = new BdbBackupper();
}
 
Example #25
Source Project: timbuctoo   Author: HuygensING   File: BdbNonPersistentEnvironmentCreator.java    License: GNU General Public License v3.0 5 votes vote down vote up
public BdbNonPersistentEnvironmentCreator() {
  configuration = new EnvironmentConfig(new Properties());
  configuration.setTransactional(true);
  configuration.setAllowCreate(true);
  configuration.setSharedCache(true);
  dbHome = Files.createTempDir();
  databases = Maps.newHashMap();
  environmentMap = Maps.newHashMap();
}
 
Example #26
Source Project: qpid-broker-j   Author: apache   File: MultiNodeTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testClusterCannotStartWithIntruder() throws Exception
{
    int intruderPort =
            new PortHelper().getNextAvailable(Arrays.stream(getBrokerAdmin().getBdbPorts()).max().getAsInt() + 1);
    String nodeName = "intruder";
    String nodeHostPort = getBrokerAdmin().getHost() + ":" + intruderPort;
    File environmentPathFile = Files.createTempDirectory("qpid-work-intruder").toFile();
    try
    {
        environmentPathFile.mkdirs();
        ReplicationConfig replicationConfig =
                new ReplicationConfig("test", nodeName, nodeHostPort);
        replicationConfig.setHelperHosts(getBrokerAdmin().getHelperHostPort());
        EnvironmentConfig envConfig = new EnvironmentConfig();
        envConfig.setAllowCreate(true);
        envConfig.setTransactional(true);
        envConfig.setDurability(new Durability(Durability.SyncPolicy.SYNC,
                                               Durability.SyncPolicy.WRITE_NO_SYNC,
                                               Durability.ReplicaAckPolicy.SIMPLE_MAJORITY));

        final String currentThreadName = Thread.currentThread().getName();
        try (ReplicatedEnvironment intruder = new ReplicatedEnvironment(environmentPathFile,
                                                                        replicationConfig,
                                                                        envConfig))
        {
            LOGGER.debug("Intruder started");
        }
        finally
        {
            Thread.currentThread().setName(currentThreadName);
        }

        Set<Integer> ports =
                Arrays.stream(getBrokerAdmin().getGroupAmqpPorts()).boxed().collect(Collectors.toSet());
        for (int port : ports)
        {
            getBrokerAdmin().awaitNodeToAttainAttributeValue(port,
                                                             BDBHAVirtualHostNode.STATE,
                                                             State.ERRORED.name());
        }

        getBrokerAdmin().stop();
        try
        {
            getBrokerAdmin().start();
            fail("Cluster cannot start with an intruder node");
        }
        catch (Exception e)
        {
            // pass
        }
    }
    finally
    {
        FileUtils.delete(environmentPathFile, true);
    }
}
 
Example #27
Source Project: qpid-broker-j   Author: apache   File: StandardEnvironmentFacade.java    License: Apache License 2.0 4 votes vote down vote up
public StandardEnvironmentFacade(StandardEnvironmentConfiguration configuration)
{
    _storePath = configuration.getStorePath();

    if (LOGGER.isInfoEnabled())
    {
        LOGGER.info("Creating environment at environment path " + _storePath);
    }

    _environmentPath = new File(_storePath);
    if (!_environmentPath.exists())
    {
        if (!_environmentPath.mkdirs())
        {
            throw new IllegalArgumentException("Environment path " + _environmentPath + " could not be read or created. "
                                               + "Ensure the path is correct and that the permissions are correct.");
        }
    }
    else if(_environmentPath.isFile())
    {
        throw new IllegalArgumentException("Environment path " + _environmentPath + " exists as a file - not a directory. "
                                           + "Ensure the path is correct.");

    }

    String name = configuration.getName();
    EnvironmentConfig envConfig = new EnvironmentConfig();
    envConfig.setAllowCreate(true);
    envConfig.setTransactional(true);
    envConfig.setCacheMode(configuration.getCacheMode());
    envConfig.setLoggingHandler(new Slf4jLoggingHandler(configuration));

    LOGGER.debug("Cache mode {}", envConfig.getCacheMode());

    Map<String, String> params = new HashMap<>(EnvironmentFacade.ENVCONFIG_DEFAULTS);
    params.putAll(configuration.getParameters());

    for (Map.Entry<String, String> configItem : params.entrySet())
    {
        if (LOGGER.isDebugEnabled())
        {
            LOGGER.debug("Setting EnvironmentConfig key "
                         + configItem.getKey()
                         + " to '"
                         + configItem.getValue()
                         + "'");
        }
        envConfig.setConfigParam(configItem.getKey(), configItem.getValue());
    }

    envConfig.setExceptionListener(new LoggingAsyncExceptionListener());

    DbInternal.setLoadPropertyFile(envConfig, false);

    File propsFile = new File(_environmentPath, "je.properties");
    if(propsFile.exists())
    {
        LOGGER.warn("The BDB configuration file at '" + _environmentPath + File.separator +  "je.properties' will NOT be loaded.  Configure BDB using Qpid context variables instead.");
    }

    EnvHomeRegistry.getInstance().registerHome(_environmentPath);

    boolean success = false;
    try
    {
        _environment = new AtomicReference<>(new Environment(_environmentPath, envConfig));
        success = true;
    }
    finally
    {
        if (!success)
        {
            EnvHomeRegistry.getInstance().deregisterHome(_environmentPath);
        }
    }

    _committer =  new CoalescingCommiter(name, this);
    _committer.start();
}
 
Example #28
Source Project: qpid-broker-j   Author: apache   File: OrphanConfigurationRecordPurger.java    License: Apache License 2.0 4 votes vote down vote up
private void purge() throws Exception
{
    EnvironmentConfig config = EnvironmentConfig.DEFAULT;
    config.setAllowCreate(false);
    config.setTransactional(true);

    try (Environment env = createEnvironment(config))
    {
        final int version = getVersion(env, READ_ONLY_DB_CONFIG);
        if (!ALLOWED_VERSIONS.contains(version))
        {
            throw new IllegalStateException(String.format("Store has unexpected version. Found %d expected %s",
                                                          version, ALLOWED_VERSIONS));
        }

        final Transaction tx = env.beginTransaction(null, TransactionConfig.DEFAULT);
        boolean success = false;
        try
        {
            purgeOrphans(env, tx);
            success = true;
        }
        finally
        {
            if (!success)
            {
                System.out.println("No config or config hierarchy records purged.");
                tx.abort();
            }
            else if (_dryRun)
            {
                System.out.println("No config or config hierarchy records purged - -dryRun flag specified.");
                tx.abort();
            }
            else
            {
                tx.commit();
                System.out.format("Config records(s) and associated hierarchy records purged.");
            }
        }
    }
}
 
Example #29
Source Project: qpid-broker-j   Author: apache   File: BDBUtils.java    License: Apache License 2.0 4 votes vote down vote up
public synchronized static void runCleaner(final Environment environment)
{
    if (environment == null || !environment.isValid())
    {
        return;
    }

    boolean cleanerWasRunning = Boolean.parseBoolean(environment.getConfig().getConfigParam(EnvironmentConfig.ENV_RUN_CLEANER));

    try
    {
        if (cleanerWasRunning)
        {
            environment.getConfig().setConfigParam(EnvironmentConfig.ENV_RUN_CLEANER, Boolean.FALSE.toString());
        }

        if (LOGGER.isDebugEnabled())
        {
            LOGGER.debug("Cleaning logs");
        }

        boolean cleaned = false;
        while (environment.cleanLog() > 0)
        {
            cleaned = true;
        }
        if (cleaned)
        {
            LOGGER.debug("Cleaned log");

            CheckpointConfig force = new CheckpointConfig();
            force.setForce(true);
            environment.checkpoint(force);

            LOGGER.debug("Checkpoint force complete");
        }
    }
    finally
    {
        if (cleanerWasRunning)
        {
            environment.getConfig().setConfigParam(EnvironmentConfig.ENV_RUN_CLEANER, Boolean.TRUE.toString());
        }
    }

}
 
Example #30
Source Project: qpid-broker-j   Author: apache   File: BDBBackup.java    License: Apache License 2.0 4 votes vote down vote up
private Environment createSourceDirEnvironment(String fromdir) throws DatabaseException
{
    // Initialize the BDB backup utility on the source directory.
    return new Environment(new File(fromdir), new EnvironmentConfig());
}