org.iq80.leveldb.WriteBatch Java Examples

@Override
public void removeApplication(ApplicationId appId)
    throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("removeApplication: appId=" + appId);
  }

  try {
    WriteBatch batch = db.createWriteBatch();
    try {
      String key = APPLICATIONS_KEY_PREFIX + appId;
      batch.delete(bytes(key));
      key = FINISHED_APPS_KEY_PREFIX + appId;
      batch.delete(bytes(key));
      db.write(batch);
    } finally {
      batch.close();
    }
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Override
public void removeLocalizedResource(String user, ApplicationId appId,
    Path localPath) throws IOException {
  String localPathStr = localPath.toString();
  String startedKey = getResourceStartedKey(user, appId, localPathStr);
  String completedKey = getResourceCompletedKey(user, appId, localPathStr);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Removing local resource at " + localPathStr);
  }
  try {
    WriteBatch batch = db.createWriteBatch();
    try {
      batch.delete(bytes(startedKey));
      batch.delete(bytes(completedKey));
      db.write(batch);
    } finally {
      batch.close();
    }
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Override
public void finishResourceLocalization(String user, ApplicationId appId,
    LocalizedResourceProto proto) throws IOException {
  String localPath = proto.getLocalPath();
  String startedKey = getResourceStartedKey(user, appId, localPath);
  String completedKey = getResourceCompletedKey(user, appId, localPath);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Storing localized resource to " + completedKey);
  }
  try {
    WriteBatch batch = db.createWriteBatch();
    try {
      batch.delete(bytes(startedKey));
      batch.put(bytes(completedKey), proto.toByteArray());
      db.write(batch);
    } finally {
      batch.close();
    }
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Override
public void removeApplication(ApplicationId appId)
    throws IOException {
  try {
    WriteBatch batch = db.createWriteBatch();
    try {
      String key = APPLICATIONS_KEY_PREFIX + appId;
      batch.delete(bytes(key));
      key = FINISHED_APPS_KEY_PREFIX + appId;
      batch.delete(bytes(key));
      db.write(batch);
    } finally {
      batch.close();
    }
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Override
public void removeContainer(ContainerId containerId)
    throws IOException {
  String keyPrefix = CONTAINERS_KEY_PREFIX + containerId.toString();
  try {
    WriteBatch batch = db.createWriteBatch();
    try {
      batch.delete(bytes(keyPrefix + CONTAINER_REQUEST_KEY_SUFFIX));
      batch.delete(bytes(keyPrefix + CONTAINER_DIAGS_KEY_SUFFIX));
      batch.delete(bytes(keyPrefix + CONTAINER_LAUNCHED_KEY_SUFFIX));
      batch.delete(bytes(keyPrefix + CONTAINER_KILLED_KEY_SUFFIX));
      batch.delete(bytes(keyPrefix + CONTAINER_EXIT_CODE_KEY_SUFFIX));
      db.write(batch);
    } finally {
      batch.close();
    }
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Override
protected void removeApplicationStateInternal(ApplicationStateData appState)
    throws IOException {
  ApplicationId appId =
      appState.getApplicationSubmissionContext().getApplicationId();
  String appKey = getApplicationNodeKey(appId);
  try {
    WriteBatch batch = db.createWriteBatch();
    try {
      batch.delete(bytes(appKey));
      for (ApplicationAttemptId attemptId : appState.attempts.keySet()) {
        String attemptKey = getApplicationAttemptNodeKey(appKey, attemptId);
        batch.delete(bytes(attemptKey));
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug("Removing state for app " + appId + " and "
            + appState.attempts.size() + " attempts" + " at " + appKey);
      }
      db.write(batch);
    } finally {
      batch.close();
    }
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Override
protected void removeApplicationStateInternal(ApplicationStateData appState)
    throws IOException {
  ApplicationId appId =
      appState.getApplicationSubmissionContext().getApplicationId();
  String appKey = getApplicationNodeKey(appId);
  try {
    WriteBatch batch = db.createWriteBatch();
    try {
      batch.delete(bytes(appKey));
      for (ApplicationAttemptId attemptId : appState.attempts.keySet()) {
        String attemptKey = getApplicationAttemptNodeKey(appKey, attemptId);
        batch.delete(bytes(attemptKey));
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug("Removing state for app " + appId + " and "
            + appState.attempts.size() + " attempts" + " at " + appKey);
      }
      db.write(batch);
    } finally {
      batch.close();
    }
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Override
public void removeLocalizedResource(String user, ApplicationId appId,
    Path localPath) throws IOException {
  String localPathStr = localPath.toString();
  String startedKey = getResourceStartedKey(user, appId, localPathStr);
  String completedKey = getResourceCompletedKey(user, appId, localPathStr);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Removing local resource at " + localPathStr);
  }
  try {
    WriteBatch batch = db.createWriteBatch();
    try {
      batch.delete(bytes(startedKey));
      batch.delete(bytes(completedKey));
      db.write(batch);
    } finally {
      batch.close();
    }
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Override
public void finishResourceLocalization(String user, ApplicationId appId,
    LocalizedResourceProto proto) throws IOException {
  String localPath = proto.getLocalPath();
  String startedKey = getResourceStartedKey(user, appId, localPath);
  String completedKey = getResourceCompletedKey(user, appId, localPath);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Storing localized resource to " + completedKey);
  }
  try {
    WriteBatch batch = db.createWriteBatch();
    try {
      batch.delete(bytes(startedKey));
      batch.put(bytes(completedKey), proto.toByteArray());
      db.write(batch);
    } finally {
      batch.close();
    }
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Override
public void removeContainer(ContainerId containerId)
    throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("removeContainer: containerId=" + containerId);
  }

  String keyPrefix = CONTAINERS_KEY_PREFIX + containerId.toString();
  try {
    WriteBatch batch = db.createWriteBatch();
    try {
      batch.delete(bytes(keyPrefix + CONTAINER_REQUEST_KEY_SUFFIX));
      batch.delete(bytes(keyPrefix + CONTAINER_DIAGS_KEY_SUFFIX));
      batch.delete(bytes(keyPrefix + CONTAINER_LAUNCHED_KEY_SUFFIX));
      batch.delete(bytes(keyPrefix + CONTAINER_KILLED_KEY_SUFFIX));
      batch.delete(bytes(keyPrefix + CONTAINER_EXIT_CODE_KEY_SUFFIX));
      db.write(batch);
    } finally {
      batch.close();
    }
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Override
public void write(WriteBatch updates) throws DBException {
  try {
    mutex.lockInterruptibly();
    pendingOps.incrementAndGet();
  } catch (InterruptedException ie) {
    throw new DBException("Interrupted while acquiring DB mutex.", ie);
  } finally {
    if (mutex.isHeldByCurrentThread()) {
      mutex.unlock();
    }
  }
  try {
    this.db.write(updates);
  } finally {
    this.pendingOps.decrementAndGet();
  }
}
 

@Override
public Snapshot write(WriteBatch updates, WriteOptions options) throws DBException {
  if (options.snapshot()) {
    throw new RuntimeException("Snapshots are unsupported.");      
  }
  
  try {
    mutex.lockInterruptibly();
    pendingOps.incrementAndGet();
  } catch (InterruptedException ie) {
    throw new DBException("Interrupted while acquiring DB mutex.", ie);
  } finally {
    if (mutex.isHeldByCurrentThread()) {
      mutex.unlock();
    }
  }
  try {
    return this.db.write(updates, options);
  } finally {
    this.pendingOps.decrementAndGet();
  }    
}
 

@Override
public void writeBatch(BatchOperation operation) throws IOException {
  List<BatchOperation.SingleOperation> operations =
      operation.getOperations();
  if (!operations.isEmpty()) {
    try (WriteBatch writeBatch = db.createWriteBatch()) {
      for (BatchOperation.SingleOperation opt : operations) {
        switch (opt.getOpt()) {
        case DELETE:
          writeBatch.delete(opt.getKey());
          break;
        case PUT:
          writeBatch.put(opt.getKey(), opt.getValue());
          break;
        default:
          throw new IllegalArgumentException("Invalid operation "
              + opt.getOpt());
        }
      }
      db.write(writeBatch);
    }
  }
}
 

private void updateByBatchInner(Map<byte[], byte[]> rows) throws Exception {
    try (WriteBatch batch = database.createWriteBatch()) {
        rows.forEach((key, value) -> {
            if (value == null) {
                batch.delete(key);
            } else {
                batch.put(key, value);
            }
        });
        database.write(batch);
    }
}
 

private void store(byte[] key, byte[] value) throws IOException {
  
  if (null == this.db) {
    return;
  }
  
  WriteBatch batch = perThreadWriteBatch.get();

  AtomicLong size = perThreadWriteBatchSize.get();
  
  boolean written = false;
  
  WriteOptions options = new WriteOptions().sync(null == key || null == value || 1.0 == syncrate);
  
  try {
    if (null != key && null != value) {
      batch.put(key, value);
      size.addAndGet(key.length + value.length);
    }
    
    if (null == key || null == value || size.get() > MAX_BATCH_SIZE) {
      
      if (syncwrites && !options.sync()) {
        options = new WriteOptions().sync(Math.random() < syncrate);
      }
      
      this.db.write(batch, options);
      size.set(0L);
      perThreadWriteBatch.remove();
      written = true;
    }
  } finally {
    if (written) {
      batch.close();
    }
  }    
}
 

private void store(List<byte[][]> kvs) throws IOException {

  //WriteBatch batch = this.db.createWriteBatch();
  
  WriteBatch batch = perThreadWriteBatch.get();

  AtomicLong size = perThreadWriteBatchSize.get();
  
  boolean written = false;
  
  try {
    if (null != kvs) {
      for (byte[][] kv: kvs) {
        batch.put(kv[0], kv[1]);
        size.addAndGet(kv[0].length + kv[1].length);
      }        
    }
    
    if (null == kvs || size.get() > MAX_ENCODER_SIZE) {
      
      WriteOptions options = new WriteOptions().sync(null == kvs || 1.0 == syncrate);
      
      if (syncwrites && !options.sync()) {
        options = new WriteOptions().sync(Math.random() < syncrate);
      }

      this.db.write(batch, options);
      size.set(0L);
      perThreadWriteBatch.remove();
      written = true;
    }
    //this.db.write(batch);
  } finally {
    if (written) {
      batch.close();
    }
  }
}
 

private void storeOrUpdateRMDT(RMDelegationTokenIdentifier tokenId,
    Long renewDate, boolean isUpdate) throws IOException {
  String tokenKey = getRMDTTokenNodeKey(tokenId);
  RMDelegationTokenIdentifierData tokenData =
      new RMDelegationTokenIdentifierData(tokenId, renewDate);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Storing token to " + tokenKey);
  }
  try {
    WriteBatch batch = db.createWriteBatch();
    try {
      batch.put(bytes(tokenKey), tokenData.toByteArray());
      if(!isUpdate) {
        ByteArrayOutputStream bs = new ByteArrayOutputStream();
        try (DataOutputStream ds = new DataOutputStream(bs)) {
          ds.writeInt(tokenId.getSequenceNumber());
        }
        if (LOG.isDebugEnabled()) {
          LOG.debug("Storing " + tokenId.getSequenceNumber() + " to "
              + RM_DT_SEQUENCE_NUMBER_KEY);   
        }
        batch.put(bytes(RM_DT_SEQUENCE_NUMBER_KEY), bs.toByteArray());
      }
      db.write(batch);
    } finally {
      batch.close();
    }
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Override
public WriteBatch createWriteBatch() {
  try {
    mutex.lockInterruptibly();
    return this.db.createWriteBatch();
  } catch (InterruptedException ie) {
    throw new RuntimeException("Interrupted while creating write batch.", ie);
  } finally {
    if (mutex.isHeldByCurrentThread()) {
      mutex.unlock();
    }      
  }
}
 

@Override
public Future<Variable> store(final Variable variable)
{
    checkNotNull(variable, "variable is null");
    checkState(!closed.get(), "already closed");
    checkState(variable instanceof JVariable, "can not process native variable, use JVariable");

    final JVariable v = (JVariable) variable;

    return executor.submit(new Callable<Variable>() {
        @Override
        public Variable call() throws Exception
        {
            final WriteOptions writeOptions = new WriteOptions();
            writeOptions.sync(true);

            final String internedName = v.getName().intern();
            synchronized (internedName) {
                final JVariable current = load(internedName);
                if (current == null || current.getUuid().equals(v.getUuid())) {
                    final JVariable update = new JVariable(internedName, v.value());
                    final WriteBatch writeBatch = db.createWriteBatch();
                    writeBatch.delete(bytes(internedName));
                    writeBatch.put(bytes(internedName), update.getEntry().toByteArray());
                    db.write(writeBatch, writeOptions);
                    return update;
                }
                else {
                    return null;
                }
            }
        }
    });
}
 

@SuppressWarnings("PMD.CloseResource")
void write(final int batchSize) throws IOException {
  final int rndByteMax = RND_MB.length - valSize;
  int rndByteOffset = 0;
  WriteBatch batch = db.createWriteBatch();
  for (int i = 0; i < keys.length; i++) {
    final int key = keys[i];
    if (intKey) {
      wkb.putInt(0, key, LITTLE_ENDIAN);
    } else {
      wkb.putStringWithoutLengthUtf8(0, padKey(key));
    }
    if (valRandom) {
      wvb.putBytes(0, RND_MB, rndByteOffset, valSize);
      rndByteOffset += valSize;
      if (rndByteOffset >= rndByteMax) {
        rndByteOffset = 0;
      }
    } else {
      wvb.putInt(0, key);
    }
    batch.put(wkb.byteArray(), wvb.byteArray());
    if (i % batchSize == 0) {
      db.write(batch);
      batch.close();
      batch = db.createWriteBatch();
    }
  }
  db.write(batch); // possible partial batch
  batch.close();
}
 

private void storeOrUpdateRMDT(RMDelegationTokenIdentifier tokenId,
    Long renewDate, boolean isUpdate) throws IOException {
  String tokenKey = getRMDTTokenNodeKey(tokenId);
  RMDelegationTokenIdentifierData tokenData =
      new RMDelegationTokenIdentifierData(tokenId, renewDate);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Storing token to " + tokenKey);
  }
  try {
    WriteBatch batch = db.createWriteBatch();
    try {
      batch.put(bytes(tokenKey), tokenData.toByteArray());
      if(!isUpdate) {
        ByteArrayOutputStream bs = new ByteArrayOutputStream();
        try (DataOutputStream ds = new DataOutputStream(bs)) {
          ds.writeInt(tokenId.getSequenceNumber());
        }
        if (LOG.isDebugEnabled()) {
          LOG.debug("Storing " + tokenId.getSequenceNumber() + " to "
              + RM_DT_SEQUENCE_NUMBER_KEY);   
        }
        batch.put(bytes(RM_DT_SEQUENCE_NUMBER_KEY), bs.toByteArray());
      }
      db.write(batch);
    } finally {
      batch.close();
    }
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 

@Override
public void removeJob(UUID uuid) throws IOException {
  try (WriteBatch wb = db.createWriteBatch()) {
    wb.delete(uuid.toString().getBytes(UTF_8));
    db.write(wb);
  }
}
 

@Override
public void updateJob(UUID uuid, JobDefinition job) throws IOException {
  try (WriteBatch wb = db.createWriteBatch()) {
    wb.put(uuid.toString().getBytes(UTF_8), MAPPER.writeValueAsBytes(job));
    db.write(wb);
  }
}
 

@Override
public final void putSilent(Buffer stream, Callback<Buffer> callback) {
    if (!isConnected) {
        throw new RuntimeException(_connectedError);
    }
    try {
        Buffer result = graph.newBuffer();
        WriteBatch batch = db.createWriteBatch();
        BufferIterator it = stream.iterator();
        boolean isFirst = true;
        while (it.hasNext()) {
            Buffer keyView = it.next();
            Buffer valueView = it.next();
            if (valueView != null) {
                batch.put(keyView.data(), valueView.data());
            }
            if (isFirst) {
                isFirst = false;
            } else {
                result.write(Constants.KEY_SEP);
            }
            result.writeAll(keyView.data());
            result.write(Constants.KEY_SEP);
            Base64.encodeLongToBuffer(HashHelper.hashBuffer(valueView, 0, valueView.length()), result);
        }
        db.write(batch);
        for (int i = 0; i < updates.size(); i++) {
            final Callback<Buffer> explicit = updates.get(i);
            explicit.on(result);
        }
        callback.on(result);
    } catch (Exception e) {
        e.printStackTrace();
        if (callback != null) {
            callback.on(null);
        }
    }
}
 

public static void batchPut() throws IOException {
    WriteBatch writeBatch = db.createWriteBatch();
    writeBatch.put("key-03".getBytes(Constants.CHARSET),"value-03".getBytes(Constants.CHARSET));
    //writeBatch.delete("key-01".getBytes(charset));
    db.write(writeBatch);
    writeBatch.close();        
}
 

private void updateByBatchInner(Map<byte[], byte[]> rows, WriteOptions options) throws Exception {
    try (WriteBatch batch = database.createWriteBatch()) {
        rows.forEach((key, value) -> {
            if (value == null) {
                batch.delete(key);
            } else {
                batch.put(key, value);
            }
        });
        database.write(batch, options);
    }
}
 

protected WriteBatch initialValue() {      
  return db.createWriteBatch();
}
 

protected WriteBatch initialValue() {      
  return db.createWriteBatch();
}
 

@Override
public long delete(WriteToken token, Metadata metadata, long start, long end) throws IOException {
  
  //
  // Regen classId/labelsId
  //
  
  // 128BITS
  metadata.setLabelsId(GTSHelper.labelsId(this.keystore.getKey(KeyStore.SIPHASH_LABELS), metadata.getLabels()));
  metadata.setClassId(GTSHelper.classId(this.keystore.getKey(KeyStore.SIPHASH_CLASS), metadata.getName()));

  //
  // Retrieve an iterator
  //
  
  DBIterator iterator = this.db.iterator();
  //
  // Seek the most recent key
  //
  
  // 128BITS
  byte[] bend = new byte[Constants.HBASE_RAW_DATA_KEY_PREFIX.length + 8 + 8 + 8];
  ByteBuffer bb = ByteBuffer.wrap(bend).order(ByteOrder.BIG_ENDIAN);
  bb.put(Constants.HBASE_RAW_DATA_KEY_PREFIX);
  bb.putLong(metadata.getClassId());
  bb.putLong(metadata.getLabelsId());
  bb.putLong(Long.MAX_VALUE - end);

  iterator.seek(bend);
  
  byte[] bstart = new byte[bend.length];
  bb = ByteBuffer.wrap(bstart).order(ByteOrder.BIG_ENDIAN);
  bb.put(Constants.HBASE_RAW_DATA_KEY_PREFIX);
  bb.putLong(metadata.getClassId());
  bb.putLong(metadata.getLabelsId());
  bb.putLong(Long.MAX_VALUE - start);
  
  //
  // Scan the iterator, deleting keys if they are between start and end
  //
  
  long count = 0L;
  
  WriteBatch batch = this.db.createWriteBatch();
  int batchsize = 0;
  
  WriteOptions options = new WriteOptions().sync(1.0 == syncrate);
              
  while (iterator.hasNext()) {
    Entry<byte[],byte[]> entry = iterator.next();

    if (Bytes.compareTo(entry.getKey(), bend) >= 0 && Bytes.compareTo(entry.getKey(), bstart) <= 0) {
      batch.delete(entry.getKey());
      batchsize++;
      
      if (MAX_DELETE_BATCHSIZE <= batchsize) {
        if (syncwrites) {
          options = new WriteOptions().sync(Math.random() < syncrate);
        }
        this.db.write(batch, options);
        batch.close();
        batch = this.db.createWriteBatch();
        batchsize = 0;
      }
      //this.db.delete(entry.getKey());
      count++;
    } else {
      break;
    }
  }
  
  if (batchsize > 0) {
    if (syncwrites) {
      options = new WriteOptions().sync(Math.random() < syncrate);
    }
    this.db.write(batch, options);
  }

  iterator.close();
  batch.close();
  
  return count;
}
 

@Override
public void put(Buffer stream, Callback<Boolean> callback) {
    if (!isConnected) {
        throw new RuntimeException(_connectedError);
    }
    try {
        Buffer result = null;
        if (updates.size() != 0) {
            result = graph.newBuffer();
        }
        WriteBatch batch = db.createWriteBatch();
        BufferIterator it = stream.iterator();
        boolean isFirst = true;
        while (it.hasNext()) {
            Buffer keyView = it.next();
            Buffer valueView = it.next();
            if (valueView != null) {
                batch.put(keyView.data(), valueView.data());
            }
            if (result != null) {
                if (isFirst) {
                    isFirst = false;
                } else {
                    result.write(Constants.KEY_SEP);
                }
                result.writeAll(keyView.data());
                result.write(Constants.KEY_SEP);
                Base64.encodeLongToBuffer(HashHelper.hashBuffer(valueView, 0, valueView.length()), result);
            }
        }
        db.write(batch);
        batch.close();
        for (int i = 0; i < updates.size(); i++) {
            final Callback<Buffer> explicit = updates.get(i);
            explicit.on(result);
        }
        if (callback != null) {
            callback.on(true);
        }
    } catch (Exception e) {
        e.printStackTrace();
        if (callback != null) {
            callback.on(false);
        }
    }
}