org.rocksdb.RocksDBException Java Examples

The following examples show how to use org.rocksdb.RocksDBException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RocksDBStore.java    From hadoop-ozone with Apache License 2.0 6 votes vote down vote up
@Override
public void writeBatch(BatchOperation operation)
    throws IOException {
  List<BatchOperation.SingleOperation> operations =
      operation.getOperations();
  if (!operations.isEmpty()) {
    try (WriteBatch writeBatch = new WriteBatch()) {
      for (BatchOperation.SingleOperation opt : operations) {
        switch (opt.getOpt()) {
        case DELETE:
          writeBatch.delete(opt.getKey());
          break;
        case PUT:
          writeBatch.put(opt.getKey(), opt.getValue());
          break;
        default:
          throw new IllegalArgumentException("Invalid operation "
              + opt.getOpt());
        }
      }
      db.write(writeOptions, writeBatch);
    } catch (RocksDBException e) {
      throw toIOException("Batch write operation failed", e);
    }
  }
}
 
Example #2
Source File: RocksDBSenseVectors.java    From biomedicus with Apache License 2.0 6 votes vote down vote up
@Override
public void removeWord(int index) {
  try (WriteBatch writeBatch = new WriteBatch()) {
    try (RocksIterator rocksIterator = rocksDB.newIterator()) {
      rocksIterator.seekToFirst();
      while (rocksIterator.isValid()) {
        SparseVector sparseVector = new SparseVector(rocksIterator.value());
        sparseVector.remove(index);
        writeBatch.put(rocksIterator.key(), sparseVector.toBytes());
      }
    }
    rocksDB.write(new WriteOptions(), writeBatch);
  } catch (RocksDBException e) {
    throw new RuntimeException(e);
  }
}
 
Example #3
Source File: RocksRawKVStore.java    From sofa-jraft with Apache License 2.0 6 votes vote down vote up
void ingestSstFiles(final EnumMap<SstColumnFamily, File> sstFileTable) {
    final Timer.Context timeCtx = getTimeContext("INGEST_SST_FILE");
    final Lock readLock = this.readWriteLock.readLock();
    readLock.lock();
    try {
        for (final Map.Entry<SstColumnFamily, File> entry : sstFileTable.entrySet()) {
            final SstColumnFamily sstColumnFamily = entry.getKey();
            final File sstFile = entry.getValue();
            final ColumnFamilyHandle columnFamilyHandle = findColumnFamilyHandle(sstColumnFamily);
            try (final IngestExternalFileOptions ingestOptions = new IngestExternalFileOptions()) {
                if (FileUtils.sizeOf(sstFile) == 0L) {
                    return;
                }
                final String filePath = sstFile.getAbsolutePath();
                LOG.info("Start ingest sst file {}.", filePath);
                this.db.ingestExternalFile(columnFamilyHandle, Collections.singletonList(filePath), ingestOptions);
            } catch (final RocksDBException e) {
                throw new StorageException("Fail to ingest sst file at path: " + sstFile, e);
            }
        }
    } finally {
        readLock.unlock();
        timeCtx.stop();
    }
}
 
Example #4
Source File: RocksDBStore.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
private String stats() {
  try {
    StringBuilder sb = new StringBuilder();
    append(sb, "rocksdb.estimate-num-keys", "Estimated Number of Keys");
    append(sb, "rocksdb.estimate-live-data-size", "Estimated Live Data Size");
    append(sb, "rocksdb.total-sst-files-size", "Total SST files size");
    append(sb, "rocksdb.estimate-pending-compaction-bytes", "Pending Compaction Bytes");

    final BlobStats blobStats = metaManager.getStats();
    if (blobStats != null) {
      blobStats.append(sb);
    }
    return sb.toString();
  } catch(RocksDBException e) {
    throw new RuntimeException(e);
  }
}
 
Example #5
Source File: RocksDBPerfTest.java    From hugegraph with Apache License 2.0 6 votes vote down vote up
@Test
public void testGet1KeyWithMultiValues() throws RocksDBException {

    put("person:1gname", "James");
    put("person:1gage", "19");
    put("person:1gcity", "Beijing");

    put("person:2gname", "Lisa");
    put("person:2gage", "20");
    put("person:2gcity", "Beijing");

    put("person:2all", "name=Lisa,age=20,city=Beijing");

    Session session = this.rocks.session();
    for (int i = 0; i < TIMES; i++) {
        s(session.get(TABLE, b("person:2all")));
    }
}
 
Example #6
Source File: RocksDbHdfsState.java    From jstorm with Apache License 2.0 6 votes vote down vote up
/**
 * Flush the data in memtable of RocksDB into disk, and then create checkpoint
 * 
 * @param batchId
 */
@Override
public void checkpoint(long batchId) {
    long startTime = System.currentTimeMillis();
    try {
        rocksDb.flush(new FlushOptions());
        Checkpoint cp = Checkpoint.create(rocksDb);
        cp.createCheckpoint(getLocalCheckpointPath(batchId));
    } catch (RocksDBException e) {
        LOG.error("Failed to create checkpoint for batch-" + batchId, e);
        throw new RuntimeException(e.getMessage());
    }

    if (JStormMetrics.enabled)
        rocksDbFlushAndCpLatency.update(System.currentTimeMillis() - startTime);
}
 
Example #7
Source File: RocksDBLookupBuilder.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public void build(ILookupTable srcLookupTable) {
    File dbFolder = new File(dbPath);
    if (dbFolder.exists()) {
        logger.info("remove rocksdb folder:{} to rebuild table cache:{}", dbPath, tableDesc.getIdentity());
        FileUtils.deleteQuietly(dbFolder);
    } else {
        logger.info("create new rocksdb folder:{} for table cache:{}", dbPath, tableDesc.getIdentity());
        dbFolder.mkdirs();
    }
    logger.info("start to build lookup table:{} to rocks db:{}", tableDesc.getIdentity(), dbPath);
    try (RocksDB rocksDB = RocksDB.open(options, dbPath)) {
        // todo use batch may improve write performance
        for (String[] row : srcLookupTable) {
            KV kv = encoder.encode(row);
            rocksDB.put(kv.getKey(), kv.getValue());
        }
    } catch (RocksDBException e) {
        logger.error("error when put data to rocksDB", e);
        throw new RuntimeException("error when write data to rocks db", e);
    }

    logger.info("source table:{} has been written to rocks db:{}", tableDesc.getIdentity(), dbPath);
}
 
Example #8
Source File: RocksDBWrapper.java    From aion with MIT License 6 votes vote down vote up
@Override
public void commit() {
    check();

    if (batch != null) {
        try {
            db.write(writeOptions, batch);
        } catch (RocksDBException e) {
            LOG.error(
                    "Unable to execute batch put/update/delete operation on "
                            + this.toString()
                            + ".",
                    e);
        }
        batch.close();
        batch = null;
    }
}
 
Example #9
Source File: RocksDBStdSessions.java    From hugegraph with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void createTable(String table) throws RocksDBException {
    if (this.cfs.containsKey(table)) {
        return;
    }

    this.checkValid();

    // Should we use options.setCreateMissingColumnFamilies() to create CF
    ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(table));
    ColumnFamilyOptions options = cfd.getOptions();
    initOptions(this.config(), null, null, options, options);
    this.cfs.put(table, new CFHandle(this.rocksdb.createColumnFamily(cfd)));

    ingestExternalFile();
}
 
Example #10
Source File: KnowledgeBase.java    From fasten with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("resource")
public static KnowledgeBase getInstance(final String kbDir, final String kbMetadataPathname, final boolean readOnly) throws RocksDBException, ClassNotFoundException, IOException {
	final boolean metadataExists = new File(kbMetadataPathname).exists();
	final boolean kbDirExists = new File(kbDir).exists();
	if (metadataExists != kbDirExists) throw new IllegalArgumentException("Either both or none of the knowledge-base directory and metadata must exist");

	RocksDB.loadLibrary();
	final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions().setCompressionType(CompressionType.LZ4_COMPRESSION);
	final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true);
	final List<ColumnFamilyDescriptor> cfDescriptors = Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions), new ColumnFamilyDescriptor(GID2URI, cfOptions), new ColumnFamilyDescriptor(URI2GID, cfOptions));

	final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
	final RocksDB db = readOnly ? RocksDB.openReadOnly(dbOptions, kbDir, cfDescriptors, columnFamilyHandles) : RocksDB.open(dbOptions, kbDir, cfDescriptors, columnFamilyHandles);

	final KnowledgeBase kb;
	if (metadataExists) {
		kb = (KnowledgeBase) BinIO.loadObject(kbMetadataPathname);
		kb.readOnly = readOnly;
		kb.callGraphDB = db;
		kb.defaultHandle = columnFamilyHandles.get(0);
		kb.gid2uriFamilyHandle = columnFamilyHandles.get(1);
		kb.uri2gidFamilyHandle = columnFamilyHandles.get(2);
	} else kb = new KnowledgeBase(db, columnFamilyHandles.get(0), columnFamilyHandles.get(1), columnFamilyHandles.get(2), kbMetadataPathname, readOnly);
	return kb;
}
 
Example #11
Source File: RocksDBIncrementalCheckpointUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Delete the record falls into [beginKeyBytes, endKeyBytes) of the db.
 *
 * @param db the target need to be clipped.
 * @param columnFamilyHandles the column family need to be clipped.
 * @param beginKeyBytes the begin key bytes
 * @param endKeyBytes the end key bytes
 */
private static void deleteRange(
	RocksDB db,
	List<ColumnFamilyHandle> columnFamilyHandles,
	byte[] beginKeyBytes,
	byte[] endKeyBytes) throws RocksDBException {

	for (ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) {
		try (RocksIteratorWrapper iteratorWrapper = RocksDBOperationUtils.getRocksIterator(db, columnFamilyHandle);
			RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(db)) {

			iteratorWrapper.seek(beginKeyBytes);

			while (iteratorWrapper.isValid()) {
				final byte[] currentKey = iteratorWrapper.key();
				if (beforeThePrefixBytes(currentKey, endKeyBytes)) {
					writeBatchWrapper.remove(columnFamilyHandle, currentKey);
				} else {
					break;
				}
				iteratorWrapper.next();
			}
		}
	}
}
 
Example #12
Source File: RocksDBDAO.java    From hudi with Apache License 2.0 6 votes vote down vote up
/**
 * Helper to load managed column family descriptors.
 */
private List<ColumnFamilyDescriptor> loadManagedColumnFamilies(DBOptions dbOptions) throws RocksDBException {
  final List<ColumnFamilyDescriptor> managedColumnFamilies = new ArrayList<>();
  final Options options = new Options(dbOptions, new ColumnFamilyOptions());
  List<byte[]> existing = RocksDB.listColumnFamilies(options, rocksDBBasePath);

  if (existing.isEmpty()) {
    LOG.info("No column family found. Loading default");
    managedColumnFamilies.add(getColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
  } else {
    LOG.info("Loading column families :" + existing.stream().map(String::new).collect(Collectors.toList()));
    managedColumnFamilies
        .addAll(existing.stream().map(RocksDBDAO::getColumnFamilyDescriptor).collect(Collectors.toList()));
  }
  return managedColumnFamilies;
}
 
Example #13
Source File: RocksDb.java    From benchmarks with Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings("PMD.CloseResource")
public void setup(final BenchmarkParams b) throws IOException {
  super.setup(b);
  wkb = new UnsafeBuffer(new byte[keySize]);
  wvb = new UnsafeBuffer(new byte[valSize]);
  loadLibrary();
  final Options options = new Options();
  options.setCreateIfMissing(true);
  options.setCompressionType(NO_COMPRESSION);
  try {
    db = open(options, tmp.getAbsolutePath());
  } catch (final RocksDBException ex) {
    throw new IOException(ex);
  }
}
 
Example #14
Source File: RocksDaoTest.java    From fasten with Apache License 2.0 6 votes vote down vote up
@Test
public void databaseTest5() throws IOException, RocksDBException {
    var json = new JSONObject("{" +
            "\"index\": 2," +
            "\"product\": \"test\"," +
            "\"version\": \"0.0.1\"," +
            "\"nodes\": [9223372036854775804, 9223372036854775805, 9223372036854775806, 9223372036854775807]," +
            "\"numInternalNodes\": 3," +
            "\"edges\": [[9223372036854775804, 9223372036854775805], [9223372036854775804, 9223372036854775807], [9223372036854775805, 9223372036854775806], [9223372036854775806, 9223372036854775807]]" +
            "}");
    var graph = GidGraph.getGraph(json);
    rocksDao.saveToRocksDb(graph.getIndex(), graph.getNodes(), graph.getNumInternalNodes(), graph.getEdges());
    var graphData = rocksDao.getGraphData(graph.getIndex());
    assertEquals(graph.getNumInternalNodes(), graphData.nodes().size() - graphData.externalNodes().size());
    assertEquals(graph.getNodes().size(), graphData.nodes().size());
    assertEquals(new LongOpenHashSet(graph.getNodes()), graphData.nodes());
    assertEquals(new LongArrayList(List.of(9223372036854775805L, 9223372036854775807L)), graphData.successors(9223372036854775804L));
    assertEquals(new LongArrayList(List.of(9223372036854775806L)), graphData.successors(9223372036854775805L));
    assertEquals(new LongArrayList(List.of(9223372036854775807L)), graphData.successors(9223372036854775806L));
    assertEquals(new LongArrayList(), graphData.predecessors(9223372036854775804L));
    assertEquals(new LongArrayList(List.of(9223372036854775804L)), graphData.predecessors(9223372036854775805L));
    assertEquals(new LongArrayList(List.of(9223372036854775805L)), graphData.predecessors(9223372036854775806L));
    assertEquals(new LongArrayList(List.of(9223372036854775804L, 9223372036854775806L)), graphData.predecessors(9223372036854775807L));
    assertEquals(graph.getEdges().size(), graphData.numArcs());
    assertEquals(new LongOpenHashSet(List.of(9223372036854775807L)), graphData.externalNodes());
}
 
Example #15
Source File: RocksDbDataStoreFactory.java    From biomedicus with Apache License 2.0 5 votes vote down vote up
@Override
public SuffixDataStore createSuffixDataStore(int id) {
  RocksDB.loadLibrary();
  try (Options options = new Options().setCreateIfMissing(true).prepareForBulkLoad()) {
    Files.createDirectories(dbPath);
    RocksDB rocksDB = RocksDB.open(options, dbPath.resolve(getSuffixesName(id)).toString());
    rocksDBS.add(rocksDB);

    return new RocksDbSuffixDataStore(rocksDB);
  } catch (RocksDBException | IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example #16
Source File: RocksDBColumnarKeyValueStorage.java    From besu with Apache License 2.0 5 votes vote down vote up
@Override
public void put(final ColumnFamilyHandle segment, final byte[] key, final byte[] value) {
  try (final OperationTimer.TimingContext ignored = metrics.getWriteLatency().startTimer()) {
    innerTx.put(segment, key, value);
  } catch (final RocksDBException e) {
    throw new StorageException(e);
  }
}
 
Example #17
Source File: RocksDBMapSet.java    From jelectrum with MIT License 5 votes vote down vote up
public void add(String key, Sha256Hash hash)
{
  String s = name + "/" + key + "/" + hash.toString();
  byte b[]=new byte[0];
  try
  {
    db.put(jdb.getWriteOption(), s.getBytes(), b);
  }
  catch(RocksDBException e)
  {
    throw new RuntimeException(e);
  }

}
 
Example #18
Source File: KnowledgeBase.java    From fasten with Apache License 2.0 5 votes vote down vote up
private long uri2GID(final FastenURI uri) {
	byte[] result;
	try {
		result = callGraphDB.get(uri2gidFamilyHandle, uri.toString().getBytes(StandardCharsets.UTF_8));
	} catch (final RocksDBException e) {
		throw new RuntimeException(e);
	}
	if (result == null) return -1;
	return Longs.fromByteArray(result);
}
 
Example #19
Source File: ByteStoreManager.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
private ByteStore newStore(String name) throws RocksDBException {
  if (inMemory) {
    return new MapStore(name);
  } else {
    final ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor(name.getBytes(UTF_8));
    ColumnFamilyHandle handle = db.createColumnFamily(columnFamilyDescriptor);
    handleIdToNameMap.put(handle.getID(), name);
    metadataManager.createEntry(name, false);

    return newRocksDBStore(name, columnFamilyDescriptor, handle);
  }
}
 
Example #20
Source File: RocksDBUtils.java    From blockchain-java with Apache License 2.0 5 votes vote down vote up
/**
 * 保存最新一个区块的Hash值
 *
 * @param tipBlockHash
 */
public void putLastBlockHash(String tipBlockHash) {
    try {
        blocksBucket.put(LAST_BLOCK_KEY, SerializeUtils.serialize(tipBlockHash));
        db.put(SerializeUtils.serialize(BLOCKS_BUCKET_KEY), SerializeUtils.serialize(blocksBucket));
    } catch (RocksDBException e) {
        log.error("Fail to put last block hash ! tipBlockHash=" + tipBlockHash, e);
        throw new RuntimeException("Fail to put last block hash ! tipBlockHash=" + tipBlockHash, e);
    }
}
 
Example #21
Source File: RocksDBDAO.java    From hudi with Apache License 2.0 5 votes vote down vote up
/**
 * Helper to add delete operation in batch.
 *
 * @param batch Batch Handle
 * @param columnFamilyName Column Family
 * @param key Key
 */
public void deleteInBatch(WriteBatch batch, String columnFamilyName, String key) {
  try {
    batch.delete(managedHandlesMap.get(columnFamilyName), key.getBytes());
  } catch (RocksDBException e) {
    throw new HoodieException(e);
  }
}
 
Example #22
Source File: RocksDBMap.java    From jelectrum with MIT License 5 votes vote down vote up
public void put(String key, ByteString value)
{
  try
  {
    String key_str = name + "/" + key;

    db.put(jdb.getWriteOption(), key_str.getBytes(), value.toByteArray());

  }
  catch(RocksDBException e)
  {
    throw new RuntimeException(e);
  }
}
 
Example #23
Source File: AbstractRocksDbHashtable.java    From ache with Apache License 2.0 5 votes vote down vote up
protected byte[] getBytes(byte[] keyBytes) {
    Preconditions.checkNotNull(this.db, "Make sure the database is open.");
    byte[] valueBytes;
    try {
        valueBytes = db.get(keyBytes);
    } catch (RocksDBException e) {
        String hexKey = BaseEncoding.base16().encode(keyBytes);
        throw new RuntimeException("Failed to get value from database for key: " + hexKey, e);
    }
    return valueBytes;
}
 
Example #24
Source File: RocksDBMap.java    From snowblossom with Apache License 2.0 5 votes vote down vote up
public void put(ByteString key, ByteString value)
{
  try
  {
    ByteString key_str = prefix.concat(key);
    db.put(jdb.getWriteOption(), key_str.toByteArray(), value.toByteArray());
  }
  catch(RocksDBException e)
  {
    throw new RuntimeException(e);
  }
}
 
Example #25
Source File: RocksDbConceptDictionary.java    From biomedicus with Apache License 2.0 5 votes vote down vote up
@Nullable
@Override
public List<ConceptRow> forNorms(StringsBag norms) {
  if (norms.uniqueTerms() == 0) {
    return null;
  }
  try {
    byte[] bytes = normsDB.get(norms.getBytes());
    return bytes == null ? null : toList(bytes);
  } catch (RocksDBException e) {
    throw new RuntimeException(e);
  }
}
 
Example #26
Source File: RocksDbDataStoreFactory.java    From biomedicus with Apache License 2.0 5 votes vote down vote up
@Override
public List<PartOfSpeech> getCandidates(String word) {
  try {
    return getPartsOfSpeechFromBytes(
        candidatesDB.get(word.getBytes(StandardCharsets.UTF_8)));
  } catch (RocksDBException e) {
    throw new RuntimeException(e);
  }
}
 
Example #27
Source File: RocksDBMapState.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void put(UK userKey, UV userValue) throws IOException, RocksDBException {

	byte[] rawKeyBytes = serializeCurrentKeyWithGroupAndNamespacePlusUserKey(userKey, userKeySerializer);
	byte[] rawValueBytes = serializeValueNullSensitive(userValue, userValueSerializer);

	backend.db.put(columnFamily, writeOptions, rawKeyBytes, rawValueBytes);
}
 
Example #28
Source File: RDB.java    From iot-mqtt with Apache License 2.0 5 votes vote down vote up
public boolean put(final ColumnFamilyHandle cfh,final WriteOptions writeOptions,final byte[] key,final byte[] value){
    try {
        this.DB.put(cfh, writeOptions, key, value);
        log.debug("[RocksDB] -> success put value");
    } catch (RocksDBException e) {
        log.error("[RocksDB] -> error while put, columnFamilyHandle:{}, key:{}, err:{}",
                cfh.isOwningHandle(), new String(key), e.getMessage(), e);
        return false;
}
    return true;
}
 
Example #29
Source File: Webapp.java    From outbackcdx with Apache License 2.0 5 votes vote down vote up
private Response getAccessRule(Web.Request req) throws IOException, Web.ResponseException, RocksDBException {
    Index index = getIndex(req);
    Long ruleId = Long.parseLong(req.param("ruleId"));
    AccessRule rule = index.accessControl.rule(ruleId);
    if (rule == null) {
        return notFound();
    }
    return jsonResponse(rule);
}
 
Example #30
Source File: OMDBUpdatesHandler.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
@Override
public void markBeginPrepare() throws RocksDBException {
  /**
   * There are no use cases yet for this method in Recon. These will be
   * implemented as and when need arises.
   */
}