Java Code Examples for org.rocksdb.RocksDB#put()

The following examples show how to use org.rocksdb.RocksDB#put() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BackupEngineTest.java    From DDMQ with Apache License 2.0 5 votes vote down vote up
/**
 * Fill database with some test values.
 *
 * @param db {@link RocksDB} instance.
 * @throws RocksDBException thrown if an error occurs within the native
 *                          part of the library.
 */
private void prepareDatabase(final RocksDB db)
        throws RocksDBException {
    db.put("key1".getBytes(), "valueV1".getBytes());
    db.put("key2".getBytes(), "valueV1".getBytes());
    db.put("key3".getBytes(), "valueV1".getBytes());
    db.put("key4".getBytes(), "valueV1".getBytes());
}
 
Example 2
Source File: BackupEngineTest.java    From DDMQ with Apache License 2.0 5 votes vote down vote up
/**
 * Fill database with some test values.
 *
 * @param db {@link RocksDB} instance.
 * @throws RocksDBException thrown if an error occurs within the native
 *                          part of the library.
 */
private void prepareDatabase(final RocksDB db)
        throws RocksDBException {
    db.put("key1".getBytes(), "valueV1".getBytes());
    db.put("key2".getBytes(), "valueV1".getBytes());
    db.put("key3".getBytes(), "valueV1".getBytes());
    db.put("key4".getBytes(), "valueV1".getBytes());
}
 
Example 3
Source File: RocksDBManager.java    From WebCollector with GNU General Public License v3.0 5 votes vote down vote up
@Override
    public void merge() throws Exception {
        LOG.info("start merge");
        RocksDB crawldbDatabase = RocksDBUtils.openCrawldbDatabase(crawlPath);


        /*合并fetch库*/
        LOG.info("merge fetch database");
        RocksDB fetchDatabase = RocksDBUtils.openFetchDatabase(crawlPath);
        RocksIterator fetchIterator = fetchDatabase.newIterator();
        for(fetchIterator.seekToFirst(); fetchIterator.isValid(); fetchIterator.next()){
            crawldbDatabase.put(fetchIterator.key(), fetchIterator.value());
        }
        fetchDatabase.close();

        /*合并link库*/
        LOG.info("merge link database");
        RocksDB linkDatabase = RocksDBUtils.openLinkDatabase(crawlPath);
        RocksIterator linkIterator = linkDatabase.newIterator();
        for(linkIterator.seekToFirst(); linkIterator.isValid(); linkIterator.next()){
            if(crawldbDatabase.get(linkIterator.key()) == null){
                crawldbDatabase.put(linkIterator.key(), linkIterator.value());
            }
        }
        linkDatabase.close();

        LOG.info("end merge");
        crawldbDatabase.close();



//        env.removeDatabase(null, "fetch");
        RocksDBUtils.destroyFetchDatabase(crawlPath);
        LOG.debug("remove fetch database");
//        env.removeDatabase(null, "link");
        RocksDBUtils.destroyLinkDatabase(crawlPath);
        LOG.debug("remove link database");

    }
 
Example 4
Source File: RocksDbUnitTest.java    From jstorm with Apache License 2.0 5 votes vote down vote up
private static void rocksDbTest(RocksDB db, List<ColumnFamilyHandle> handlers) {
    try {
        ColumnFamilyHandle handler1 = null;
        ColumnFamilyHandle handler2 = null;
        if (handlers.size() > 0) {
            // skip default column family
            handler1 = handlers.get(1);
            handler2 = handlers.get(2);
        } else {
            handler1 = db.createColumnFamily(new ColumnFamilyDescriptor("test1".getBytes()));
            handler2 = db.createColumnFamily(new ColumnFamilyDescriptor("test2".getBytes()));
        }
        int startValue1 = getStartValue(db, handler1);
        int startValue2 = getStartValue(db, handler2);;

        Checkpoint cp = Checkpoint.create(db);
   
        if (isCompaction) {
            db.compactRange();
            LOG.info("Compaction!");
        }

        long flushWaitTime = System.currentTimeMillis() + flushInterval;
        for (int i = 0; i < putNum || putNum == -1; i++) {
            db.put(handler1, String.valueOf(i % 1000).getBytes(), String.valueOf(startValue1 + i).getBytes());
            db.put(handler2, String.valueOf(i % 1000).getBytes(), String.valueOf(startValue2 + i).getBytes());
            if (isFlush && flushWaitTime <= System.currentTimeMillis()) {
                db.flush(new FlushOptions());
                if (isCheckpoint) {
                    cp.createCheckpoint(cpPath + "/" + i);
                }
                flushWaitTime = System.currentTimeMillis() + flushInterval;
            }
        }
    } catch (RocksDBException e) {
        LOG.error("Failed to put or flush", e);
    }
}
 
Example 5
Source File: BackupEngineTest.java    From DDMQ with Apache License 2.0 4 votes vote down vote up
@Test
public void restoreLatestBackup() throws RocksDBException {
    try (final Options opt = new Options().setCreateIfMissing(true)) {
        // Open empty database.
        RocksDB db = null;
        try {
            db = RocksDB.open(opt,
                    dbFolder.getRoot().getAbsolutePath());
            // Fill database with some test values
            prepareDatabase(db);

            try (final BackupableDBOptions bopt = new BackupableDBOptions(
                    backupFolder.getRoot().getAbsolutePath());
                 final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) {
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 1);
                db.put("key1".getBytes(), "valueV2".getBytes());
                db.put("key2".getBytes(), "valueV2".getBytes());
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 2);
                db.put("key1".getBytes(), "valueV3".getBytes());
                db.put("key2".getBytes(), "valueV3".getBytes());
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V3");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V3");

                db.close();
                db = null;

                verifyNumberOfValidBackups(be, 2);
                // restore db from latest backup
                try (final RestoreOptions ropts = new RestoreOptions(false)) {
                    be.restoreDbFromLatestBackup(dbFolder.getRoot().getAbsolutePath(),
                            dbFolder.getRoot().getAbsolutePath(), ropts);
                }

                // Open database again.
                db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath());

                // Values must have suffix V2 because of restoring latest backup.
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V2");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V2");
            }
        } finally {
            if (db != null) {
                db.close();
            }
        }
    }
}
 
Example 6
Source File: BackupEngineTest.java    From DDMQ with Apache License 2.0 4 votes vote down vote up
@Test
public void restoreFromBackup()
        throws RocksDBException {
    try (final Options opt = new Options().setCreateIfMissing(true)) {
        RocksDB db = null;
        try {
            // Open empty database.
            db = RocksDB.open(opt,
                    dbFolder.getRoot().getAbsolutePath());
            // Fill database with some test values
            prepareDatabase(db);
            try (final BackupableDBOptions bopt = new BackupableDBOptions(
                    backupFolder.getRoot().getAbsolutePath());
                 final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) {
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 1);
                db.put("key1".getBytes(), "valueV2".getBytes());
                db.put("key2".getBytes(), "valueV2".getBytes());
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 2);
                db.put("key1".getBytes(), "valueV3".getBytes());
                db.put("key2".getBytes(), "valueV3".getBytes());
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V3");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V3");

                //close the database
                db.close();
                db = null;

                //restore the backup
                final List<BackupInfo> backupInfo = verifyNumberOfValidBackups(be, 2);
                // restore db from first backup
                be.restoreDbFromBackup(backupInfo.get(0).backupId(),
                        dbFolder.getRoot().getAbsolutePath(),
                        dbFolder.getRoot().getAbsolutePath(),
                        new RestoreOptions(false));
                // Open database again.
                db = RocksDB.open(opt,
                        dbFolder.getRoot().getAbsolutePath());
                // Values must have suffix V2 because of restoring latest backup.
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V1");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V1");
            }
        } finally {
            if (db != null) {
                db.close();
            }
        }
    }
}
 
Example 7
Source File: BackupEngineTest.java    From DDMQ with Apache License 2.0 4 votes vote down vote up
@Test
public void restoreLatestBackup() throws RocksDBException {
    try (final Options opt = new Options().setCreateIfMissing(true)) {
        // Open empty database.
        RocksDB db = null;
        try {
            db = RocksDB.open(opt,
                    dbFolder.getRoot().getAbsolutePath());
            // Fill database with some test values
            prepareDatabase(db);

            try (final BackupableDBOptions bopt = new BackupableDBOptions(
                    backupFolder.getRoot().getAbsolutePath());
                 final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) {
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 1);
                db.put("key1".getBytes(), "valueV2".getBytes());
                db.put("key2".getBytes(), "valueV2".getBytes());
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 2);
                db.put("key1".getBytes(), "valueV3".getBytes());
                db.put("key2".getBytes(), "valueV3".getBytes());
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V3");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V3");

                db.close();
                db = null;

                verifyNumberOfValidBackups(be, 2);
                // restore db from latest backup
                try (final RestoreOptions ropts = new RestoreOptions(false)) {
                    be.restoreDbFromLatestBackup(dbFolder.getRoot().getAbsolutePath(),
                            dbFolder.getRoot().getAbsolutePath(), ropts);
                }

                // Open database again.
                db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath());

                // Values must have suffix V2 because of restoring latest backup.
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V2");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V2");
            }
        } finally {
            if (db != null) {
                db.close();
            }
        }
    }
}
 
Example 8
Source File: BackupEngineTest.java    From DDMQ with Apache License 2.0 4 votes vote down vote up
@Test
public void restoreFromBackup()
        throws RocksDBException {
    try (final Options opt = new Options().setCreateIfMissing(true)) {
        RocksDB db = null;
        try {
            // Open empty database.
            db = RocksDB.open(opt,
                    dbFolder.getRoot().getAbsolutePath());
            // Fill database with some test values
            prepareDatabase(db);
            try (final BackupableDBOptions bopt = new BackupableDBOptions(
                    backupFolder.getRoot().getAbsolutePath());
                 final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) {
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 1);
                db.put("key1".getBytes(), "valueV2".getBytes());
                db.put("key2".getBytes(), "valueV2".getBytes());
                be.createNewBackup(db, true);
                verifyNumberOfValidBackups(be, 2);
                db.put("key1".getBytes(), "valueV3".getBytes());
                db.put("key2".getBytes(), "valueV3".getBytes());
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V3");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V3");

                //close the database
                db.close();
                db = null;

                //restore the backup
                final List<BackupInfo> backupInfo = verifyNumberOfValidBackups(be, 2);
                // restore db from first backup
                be.restoreDbFromBackup(backupInfo.get(0).backupId(),
                        dbFolder.getRoot().getAbsolutePath(),
                        dbFolder.getRoot().getAbsolutePath(),
                        new RestoreOptions(false));
                // Open database again.
                db = RocksDB.open(opt,
                        dbFolder.getRoot().getAbsolutePath());
                // Values must have suffix V2 because of restoring latest backup.
                assertThat(new String(db.get("key1".getBytes()))).endsWith("V1");
                assertThat(new String(db.get("key2".getBytes()))).endsWith("V1");
            }
        } finally {
            if (db != null) {
                db.close();
            }
        }
    }
}
 
Example 9
Source File: PubchemTTLMerger.java    From act with GNU General Public License v3.0 4 votes vote down vote up
protected void merge(Pair<RocksDB, Map<COLUMN_FAMILIES, ColumnFamilyHandle>> dbAndHandles)
    throws RocksDBException, IOException, ClassNotFoundException {
  LOGGER.info("Beginning merge on Pubchem CID");
  RocksDB db = dbAndHandles.getLeft();
  ColumnFamilyHandle pubchemIdCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.CID_TO_HASHES);
  ColumnFamilyHandle meshCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.HASH_TO_MESH);
  ColumnFamilyHandle synonymCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.HASH_TO_SYNONYMS);
  ColumnFamilyHandle synonymTypeCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.HASH_TO_SYNONYM_TYPE);
  ColumnFamilyHandle mergeResultsCFH = dbAndHandles.getRight().get(COLUMN_FAMILIES.CID_TO_SYNONYMS);

  RocksIterator cidIterator = db.newIterator(pubchemIdCFH);
  // With help from https://github.com/facebook/rocksdb/wiki/Basic-Operations
  int processed = 0;
  for (cidIterator.seekToFirst(); cidIterator.isValid(); cidIterator.next()) {
    byte[] key = cidIterator.key();
    byte[] val = cidIterator.value();
    String pubchemId = new String(key, UTF8);
    List<String> hashes;
    try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(val))) {
      // We know all our values so far have been lists of strings, so this should be completely safe.
      hashes = (List<String>) ois.readObject();
    }

    PubchemSynonyms pubchemSynonyms = new PubchemSynonyms(pubchemId);

    /* The hash keys are based on synonym value, which we can manually compute with:
     *   $ echo -n  'dimethyltin(iv)' | md5
     * This means that MeSH ids are linked to synonyms rather than pubchem ids.  We need to look up each cid-linked
     * hash in both the MeSH and synonym collections, as the key may legitimately exist in both (and serve to link
     * cid to synonym and cid to MeSH). */
    for (String hash : hashes) {
      /* Note: these ids are not proper MeSH topic ids, but are internal MeSH ids found in the RDF and TTL
       * representations of the MeSH corpus.  You can find them in the MeSH .nt or .xml files, but they won't turn up
       * anything on the MeSH website. */
      List<String> meshIds = getValueAsObject(db, meshCFH, hash);
      if (meshIds != null) {
        pubchemSynonyms.addMeSHIds(meshIds);
      }

      List<String> synonyms = getValueAsObject(db, synonymCFH, hash);
      // There are, surprisingly, some dangling hashes in the DB!  Handle them gracefully.
      if (synonyms == null) {
        LOGGER.warn("Dangling synonym hash reference, adding empty list in place of value: cid = %s, hash = %s",
            pubchemId, hash);
        synonyms = Collections.emptyList();
      }

      List<String> synonymTypeStrings = getValueAsObject(db, synonymTypeCFH, hash);
      Set<PC_SYNONYM_TYPES> synonymTypes = DEFAULT_SYNONYM_DATA_TYPES;
      if (synonymTypeStrings != null) {
        synonymTypes = synonymTypeStrings.stream().map(PC_SYNONYM_TYPES::valueOf).collect(Collectors.toSet());
      }

      if (synonymTypes.size() == 0) {
        LOGGER.warn("Found zero synonym types for synonym, defaulting to %s: %s %s, synonyms = %s",
            PC_SYNONYM_TYPES.UNKNOWN.name(), pubchemId, hash, StringUtils.join(synonyms, ", "));
      }
      /* It turns out that *lots* of synonyms are duplicated as depositor supplied names, so don't complain about it
       * here.  For performance sake we might want to consider changing the data model of PubchemSynonyms to reduce
       * synonym string duplication, as the current model is pretty inefficient. */

      for (PC_SYNONYM_TYPES synonymType : synonymTypes) {
        for (String synonym : synonyms) {
          // Let the PubchemSynonyms object do the de-duplication for us rather than reducing `synonyms` to a Set.
          pubchemSynonyms.addSynonym(synonymType, synonym);
        }
      }
    }

    try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
         ObjectOutputStream oo = new ObjectOutputStream(bos)) {
      oo.writeObject(pubchemSynonyms);
      oo.flush();

      db.put(mergeResultsCFH, key, bos.toByteArray());
    }

    processed++;
    if (processed % 100000 == 0) {
      LOGGER.info("Merged %d entries on Pubchem compound id", processed);
    }
  }
  LOGGER.info("Merge complete, %d entries processed", processed);
}
 
Example 10
Source File: RocksDBUtils.java    From WebCollector with GNU General Public License v3.0 4 votes vote down vote up
public static void put(RocksDB rocksDB, String key,String value) throws UnsupportedEncodingException, RocksDBException {
    rocksDB.put(strToKeyOrValue(key), strToKeyOrValue(value));
}