Java Code Examples for org.apache.hadoop.yarn.server.utils.LeveldbIterator

The following examples show how to use org.apache.hadoop.yarn.server.utils.LeveldbIterator. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: NMLeveldbStateStoreService.java    License: Apache License 2.0 6 votes vote down vote up
private LocalResourceTrackerState loadResourceTrackerState(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  final String completedPrefix = keyPrefix + LOCALIZATION_COMPLETED_SUFFIX;
  final String startedPrefix = keyPrefix + LOCALIZATION_STARTED_SUFFIX;
  LocalResourceTrackerState state = new LocalResourceTrackerState();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (key.startsWith(completedPrefix)) {
      state.localizedResources = loadCompletedResources(iter,
          completedPrefix);
    } else if (key.startsWith(startedPrefix)) {
      state.inProgressResources = loadStartedResources(iter, startedPrefix);
    } else {
      throw new IOException("Unexpected key in resource tracker state: "
          + key);
    }
  }

  return state;
}
 
Example 2
Source Project: hadoop   Source File: NMLeveldbStateStoreService.java    License: Apache License 2.0 6 votes vote down vote up
private List<LocalizedResourceProto> loadCompletedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  List<LocalizedResourceProto> rsrcs =
      new ArrayList<LocalizedResourceProto>();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("Loading completed resource from " + key);
    }
    rsrcs.add(LocalizedResourceProto.parseFrom(entry.getValue()));
    iter.next();
  }

  return rsrcs;
}
 
Example 3
Source Project: hadoop   Source File: NMLeveldbStateStoreService.java    License: Apache License 2.0 6 votes vote down vote up
private Map<LocalResourceProto, Path> loadStartedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  Map<LocalResourceProto, Path> rsrcs =
      new HashMap<LocalResourceProto, Path>();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    Path localPath = new Path(key.substring(keyPrefix.length()));
    if (LOG.isDebugEnabled()) {
      LOG.debug("Loading in-progress resource at " + localPath);
    }
    rsrcs.put(LocalResourceProto.parseFrom(entry.getValue()), localPath);
    iter.next();
  }

  return rsrcs;
}
 
Example 4
Source Project: hadoop   Source File: LeveldbTimelineStateStore.java    License: Apache License 2.0 6 votes vote down vote up
private int loadTokenMasterKeys(TimelineServiceState state)
    throws IOException {
  byte[] base = KeyBuilder.newInstance().add(TOKEN_MASTER_KEY_ENTRY_PREFIX)
      .getBytesForLookup();
  int numKeys = 0;
  LeveldbIterator iterator = null;
  try {
    for (iterator = new LeveldbIterator(db), iterator.seek(base);
        iterator.hasNext(); iterator.next()) {
      byte[] k = iterator.peekNext().getKey();
      if (!prefixMatches(base, base.length, k)) {
        break;
      }
      byte[] v = iterator.peekNext().getValue();
      loadTokenMasterKeyData(state, v);
      ++numKeys;
    }
  } finally {
    IOUtils.cleanup(LOG, iterator);
  }
  return numKeys;
}
 
Example 5
Source Project: hadoop   Source File: LeveldbTimelineStateStore.java    License: Apache License 2.0 6 votes vote down vote up
private int loadTokens(TimelineServiceState state) throws IOException {
  byte[] base = KeyBuilder.newInstance().add(TOKEN_ENTRY_PREFIX)
      .getBytesForLookup();
  int numTokens = 0;
  LeveldbIterator iterator = null;
  try {
    for (iterator = new LeveldbIterator(db), iterator.seek(base);
        iterator.hasNext(); iterator.next()) {
      byte[] k = iterator.peekNext().getKey();
      if (!prefixMatches(base, base.length, k)) {
        break;
      }
      byte[] v = iterator.peekNext().getValue();
      loadTokenData(state, v);
      ++numTokens;
    }
  } catch (DBException e) {
    throw new IOException(e);
  } finally {
    IOUtils.cleanup(LOG, iterator);
  }
  return numTokens;
}
 
Example 6
Source Project: big-c   Source File: NMLeveldbStateStoreService.java    License: Apache License 2.0 6 votes vote down vote up
private LocalResourceTrackerState loadResourceTrackerState(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  final String completedPrefix = keyPrefix + LOCALIZATION_COMPLETED_SUFFIX;
  final String startedPrefix = keyPrefix + LOCALIZATION_STARTED_SUFFIX;
  LocalResourceTrackerState state = new LocalResourceTrackerState();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (key.startsWith(completedPrefix)) {
      state.localizedResources = loadCompletedResources(iter,
          completedPrefix);
    } else if (key.startsWith(startedPrefix)) {
      state.inProgressResources = loadStartedResources(iter, startedPrefix);
    } else {
      throw new IOException("Unexpected key in resource tracker state: "
          + key);
    }
  }

  return state;
}
 
Example 7
Source Project: big-c   Source File: NMLeveldbStateStoreService.java    License: Apache License 2.0 6 votes vote down vote up
private List<LocalizedResourceProto> loadCompletedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  List<LocalizedResourceProto> rsrcs =
      new ArrayList<LocalizedResourceProto>();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("Loading completed resource from " + key);
    }
    rsrcs.add(LocalizedResourceProto.parseFrom(entry.getValue()));
    iter.next();
  }

  return rsrcs;
}
 
Example 8
Source Project: big-c   Source File: NMLeveldbStateStoreService.java    License: Apache License 2.0 6 votes vote down vote up
private Map<LocalResourceProto, Path> loadStartedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  Map<LocalResourceProto, Path> rsrcs =
      new HashMap<LocalResourceProto, Path>();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    Path localPath = new Path(key.substring(keyPrefix.length()));
    if (LOG.isDebugEnabled()) {
      LOG.debug("Loading in-progress resource at " + localPath);
    }
    rsrcs.put(LocalResourceProto.parseFrom(entry.getValue()), localPath);
    iter.next();
  }

  return rsrcs;
}
 
Example 9
Source Project: big-c   Source File: LeveldbTimelineStateStore.java    License: Apache License 2.0 6 votes vote down vote up
private int loadTokenMasterKeys(TimelineServiceState state)
    throws IOException {
  byte[] base = KeyBuilder.newInstance().add(TOKEN_MASTER_KEY_ENTRY_PREFIX)
      .getBytesForLookup();
  int numKeys = 0;
  LeveldbIterator iterator = null;
  try {
    for (iterator = new LeveldbIterator(db), iterator.seek(base);
        iterator.hasNext(); iterator.next()) {
      byte[] k = iterator.peekNext().getKey();
      if (!prefixMatches(base, base.length, k)) {
        break;
      }
      byte[] v = iterator.peekNext().getValue();
      loadTokenMasterKeyData(state, v);
      ++numKeys;
    }
  } finally {
    IOUtils.cleanup(LOG, iterator);
  }
  return numKeys;
}
 
Example 10
Source Project: big-c   Source File: LeveldbTimelineStateStore.java    License: Apache License 2.0 6 votes vote down vote up
private int loadTokens(TimelineServiceState state) throws IOException {
  byte[] base = KeyBuilder.newInstance().add(TOKEN_ENTRY_PREFIX)
      .getBytesForLookup();
  int numTokens = 0;
  LeveldbIterator iterator = null;
  try {
    for (iterator = new LeveldbIterator(db), iterator.seek(base);
        iterator.hasNext(); iterator.next()) {
      byte[] k = iterator.peekNext().getKey();
      if (!prefixMatches(base, base.length, k)) {
        break;
      }
      byte[] v = iterator.peekNext().getValue();
      loadTokenData(state, v);
      ++numTokens;
    }
  } catch (DBException e) {
    throw new IOException(e);
  } finally {
    IOUtils.cleanup(LOG, iterator);
  }
  return numTokens;
}
 
Example 11
Source Project: hadoop   Source File: NMLeveldbStateStoreService.java    License: Apache License 2.0 5 votes vote down vote up
private RecoveredContainerState loadContainerState(ContainerId containerId,
    LeveldbIterator iter, String keyPrefix) throws IOException {
  RecoveredContainerState rcs = new RecoveredContainerState();
  rcs.status = RecoveredContainerStatus.REQUESTED;
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }
    iter.next();

    String suffix = key.substring(keyPrefix.length()-1);  // start with '/'
    if (suffix.equals(CONTAINER_REQUEST_KEY_SUFFIX)) {
      rcs.startRequest = new StartContainerRequestPBImpl(
          StartContainerRequestProto.parseFrom(entry.getValue()));
    } else if (suffix.equals(CONTAINER_DIAGS_KEY_SUFFIX)) {
      rcs.diagnostics = asString(entry.getValue());
    } else if (suffix.equals(CONTAINER_LAUNCHED_KEY_SUFFIX)) {
      if (rcs.status == RecoveredContainerStatus.REQUESTED) {
        rcs.status = RecoveredContainerStatus.LAUNCHED;
      }
    } else if (suffix.equals(CONTAINER_KILLED_KEY_SUFFIX)) {
      rcs.killed = true;
    } else if (suffix.equals(CONTAINER_EXIT_CODE_KEY_SUFFIX)) {
      rcs.status = RecoveredContainerStatus.COMPLETED;
      rcs.exitCode = Integer.parseInt(asString(entry.getValue()));
    } else {
      throw new IOException("Unexpected container state key: " + key);
    }
  }
  return rcs;
}
 
Example 12
Source Project: hadoop   Source File: NMLeveldbStateStoreService.java    License: Apache License 2.0 5 votes vote down vote up
private RecoveredUserResources loadUserLocalizedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  RecoveredUserResources userResources = new RecoveredUserResources();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (key.startsWith(LOCALIZATION_FILECACHE_SUFFIX, keyPrefix.length())) {
      userResources.privateTrackerState = loadResourceTrackerState(iter,
          keyPrefix + LOCALIZATION_FILECACHE_SUFFIX);
    } else if (key.startsWith(LOCALIZATION_APPCACHE_SUFFIX,
        keyPrefix.length())) {
      int appIdStartPos = keyPrefix.length() +
          LOCALIZATION_APPCACHE_SUFFIX.length();
      int appIdEndPos = key.indexOf('/', appIdStartPos);
      if (appIdEndPos < 0) {
        throw new IOException("Unable to determine appID in resource key: "
            + key);
      }
      ApplicationId appId = ConverterUtils.toApplicationId(
          key.substring(appIdStartPos, appIdEndPos));
      userResources.appTrackerStates.put(appId,
          loadResourceTrackerState(iter, key.substring(0, appIdEndPos+1)));
    } else {
      throw new IOException("Unexpected user resource key " + key);
    }
  }
  return userResources;
}
 
Example 13
Source Project: hadoop   Source File: LeveldbRMStateStore.java    License: Apache License 2.0 5 votes vote down vote up
private int loadRMApp(RMState rmState, LeveldbIterator iter, String appIdStr,
    byte[] appData) throws IOException {
  ApplicationStateData appState = createApplicationState(appIdStr, appData);
  ApplicationId appId =
      appState.getApplicationSubmissionContext().getApplicationId();
  rmState.appState.put(appId, appState);
  String attemptNodePrefix = getApplicationNodeKey(appId) + SEPARATOR;
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(attemptNodePrefix)) {
      break;
    }

    String attemptId = key.substring(attemptNodePrefix.length());
    if (attemptId.startsWith(ApplicationAttemptId.appAttemptIdStrPrefix)) {
      ApplicationAttemptStateData attemptState =
          createAttemptState(attemptId, entry.getValue());
      appState.attempts.put(attemptState.getAttemptId(), attemptState);
    } else {
      LOG.warn("Ignoring unknown application key: " + key);
    }
    iter.next();
  }
  int numAttempts = appState.attempts.size();
  if (LOG.isDebugEnabled()) {
    LOG.debug("Loaded application " + appId + " with " + numAttempts
        + " attempts");
  }
  return numAttempts;
}
 
Example 14
Source Project: hadoop   Source File: LeveldbTimelineStore.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
List<String> getEntityTypes() throws IOException {
  LeveldbIterator iterator = null;
  try {
    iterator = getDbIterator(false);
    List<String> entityTypes = new ArrayList<String>();
    iterator.seek(ENTITY_ENTRY_PREFIX);
    while (iterator.hasNext()) {
      byte[] key = iterator.peekNext().getKey();
      if (key[0] != ENTITY_ENTRY_PREFIX[0]) {
        break;
      }
      KeyParser kp = new KeyParser(key,
          ENTITY_ENTRY_PREFIX.length);
      String entityType = kp.getNextString();
      entityTypes.add(entityType);
      byte[] lookupKey = KeyBuilder.newInstance().add(ENTITY_ENTRY_PREFIX)
          .add(entityType).getBytesForLookup();
      if (lookupKey[lookupKey.length - 1] != 0x0) {
        throw new IOException("Found unexpected end byte in lookup key");
      }
      lookupKey[lookupKey.length - 1] = 0x1;
      iterator.seek(lookupKey);
    }
    return entityTypes;
  } catch(DBException e) {
    throw new IOException(e);            	
  } finally {
    IOUtils.cleanup(LOG, iterator);
  }
}
 
Example 15
Source Project: hadoop   Source File: LeveldbTimelineStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Finds all keys in the db that have a given prefix and deletes them on
 * the given write batch.
 */
private void deleteKeysWithPrefix(WriteBatch writeBatch, byte[] prefix,
    LeveldbIterator iterator) {
  for (iterator.seek(prefix); iterator.hasNext(); iterator.next()) {
    byte[] key = iterator.peekNext().getKey();
    if (!prefixMatches(prefix, prefix.length, key)) {
      break;
    }
    writeBatch.delete(key);
  }
}
 
Example 16
Source Project: hadoop   Source File: LeveldbTimelineStore.java    License: Apache License 2.0 5 votes vote down vote up
private static TimelineDomain getTimelineDomain(
    LeveldbIterator iterator, String domainId, byte[] prefix) throws IOException {
  // Iterate over all the rows whose key starts with prefix to retrieve the
  // domain information.
  TimelineDomain domain = new TimelineDomain();
  domain.setId(domainId);
  boolean noRows = true;
  for (; iterator.hasNext(); iterator.next()) {
    byte[] key = iterator.peekNext().getKey();
    if (!prefixMatches(prefix, prefix.length, key)) {
      break;
    }
    if (noRows) {
      noRows = false;
    }
    byte[] value = iterator.peekNext().getValue();
    if (value != null && value.length > 0) {
      if (key[prefix.length] == DESCRIPTION_COLUMN[0]) {
        domain.setDescription(new String(value, Charset.forName("UTF-8")));
      } else if (key[prefix.length] == OWNER_COLUMN[0]) {
        domain.setOwner(new String(value, Charset.forName("UTF-8")));
      } else if (key[prefix.length] == READER_COLUMN[0]) {
        domain.setReaders(new String(value, Charset.forName("UTF-8")));
      } else if (key[prefix.length] == WRITER_COLUMN[0]) {
        domain.setWriters(new String(value, Charset.forName("UTF-8")));
      } else if (key[prefix.length] == TIMESTAMP_COLUMN[0]) {
        domain.setCreatedTime(readReverseOrderedLong(value, 0));
        domain.setModifiedTime(readReverseOrderedLong(value, 8));
      } else {
        LOG.error("Unrecognized domain column: " + key[prefix.length]);
      }
    }
  }
  if (noRows) {
    return null;
  } else {
    return domain;
  }
}
 
Example 17
Source Project: hadoop   Source File: TestLeveldbTimelineStore.java    License: Apache License 2.0 5 votes vote down vote up
private boolean deleteNextEntity(String entityType, byte[] ts)
    throws IOException, InterruptedException {
  LeveldbIterator iterator = null;
  LeveldbIterator pfIterator = null;
  try {
    iterator = ((LeveldbTimelineStore)store).getDbIterator(false);
    pfIterator = ((LeveldbTimelineStore)store).getDbIterator(false);
    return ((LeveldbTimelineStore)store).deleteNextEntity(entityType, ts,
        iterator, pfIterator, false);
  } catch(DBException e) {
    throw new IOException(e);   	
  } finally {
    IOUtils.cleanup(null, iterator, pfIterator);
  }
}
 
Example 18
Source Project: big-c   Source File: NMLeveldbStateStoreService.java    License: Apache License 2.0 5 votes vote down vote up
private RecoveredContainerState loadContainerState(ContainerId containerId,
    LeveldbIterator iter, String keyPrefix) throws IOException {
  RecoveredContainerState rcs = new RecoveredContainerState();
  rcs.status = RecoveredContainerStatus.REQUESTED;
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }
    iter.next();

    String suffix = key.substring(keyPrefix.length()-1);  // start with '/'
    if (suffix.equals(CONTAINER_REQUEST_KEY_SUFFIX)) {
      rcs.startRequest = new StartContainerRequestPBImpl(
          StartContainerRequestProto.parseFrom(entry.getValue()));
    } else if (suffix.equals(CONTAINER_DIAGS_KEY_SUFFIX)) {
      rcs.diagnostics = asString(entry.getValue());
    } else if (suffix.equals(CONTAINER_LAUNCHED_KEY_SUFFIX)) {
      if (rcs.status == RecoveredContainerStatus.REQUESTED) {
        rcs.status = RecoveredContainerStatus.LAUNCHED;
      }
    } else if (suffix.equals(CONTAINER_KILLED_KEY_SUFFIX)) {
      rcs.killed = true;
    } else if (suffix.equals(CONTAINER_EXIT_CODE_KEY_SUFFIX)) {
      rcs.status = RecoveredContainerStatus.COMPLETED;
      rcs.exitCode = Integer.parseInt(asString(entry.getValue()));
    } else {
      throw new IOException("Unexpected container state key: " + key);
    }
  }
  return rcs;
}
 
Example 19
Source Project: big-c   Source File: NMLeveldbStateStoreService.java    License: Apache License 2.0 5 votes vote down vote up
private RecoveredUserResources loadUserLocalizedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  RecoveredUserResources userResources = new RecoveredUserResources();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (key.startsWith(LOCALIZATION_FILECACHE_SUFFIX, keyPrefix.length())) {
      userResources.privateTrackerState = loadResourceTrackerState(iter,
          keyPrefix + LOCALIZATION_FILECACHE_SUFFIX);
    } else if (key.startsWith(LOCALIZATION_APPCACHE_SUFFIX,
        keyPrefix.length())) {
      int appIdStartPos = keyPrefix.length() +
          LOCALIZATION_APPCACHE_SUFFIX.length();
      int appIdEndPos = key.indexOf('/', appIdStartPos);
      if (appIdEndPos < 0) {
        throw new IOException("Unable to determine appID in resource key: "
            + key);
      }
      ApplicationId appId = ConverterUtils.toApplicationId(
          key.substring(appIdStartPos, appIdEndPos));
      userResources.appTrackerStates.put(appId,
          loadResourceTrackerState(iter, key.substring(0, appIdEndPos+1)));
    } else {
      throw new IOException("Unexpected user resource key " + key);
    }
  }
  return userResources;
}
 
Example 20
Source Project: big-c   Source File: LeveldbRMStateStore.java    License: Apache License 2.0 5 votes vote down vote up
private int loadRMApp(RMState rmState, LeveldbIterator iter, String appIdStr,
    byte[] appData) throws IOException {
  ApplicationStateData appState = createApplicationState(appIdStr, appData);
  ApplicationId appId =
      appState.getApplicationSubmissionContext().getApplicationId();
  rmState.appState.put(appId, appState);
  String attemptNodePrefix = getApplicationNodeKey(appId) + SEPARATOR;
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(attemptNodePrefix)) {
      break;
    }

    String attemptId = key.substring(attemptNodePrefix.length());
    if (attemptId.startsWith(ApplicationAttemptId.appAttemptIdStrPrefix)) {
      ApplicationAttemptStateData attemptState =
          createAttemptState(attemptId, entry.getValue());
      appState.attempts.put(attemptState.getAttemptId(), attemptState);
    } else {
      LOG.warn("Ignoring unknown application key: " + key);
    }
    iter.next();
  }
  int numAttempts = appState.attempts.size();
  if (LOG.isDebugEnabled()) {
    LOG.debug("Loaded application " + appId + " with " + numAttempts
        + " attempts");
  }
  return numAttempts;
}
 
Example 21
Source Project: big-c   Source File: LeveldbTimelineStore.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
List<String> getEntityTypes() throws IOException {
  LeveldbIterator iterator = null;
  try {
    iterator = getDbIterator(false);
    List<String> entityTypes = new ArrayList<String>();
    iterator.seek(ENTITY_ENTRY_PREFIX);
    while (iterator.hasNext()) {
      byte[] key = iterator.peekNext().getKey();
      if (key[0] != ENTITY_ENTRY_PREFIX[0]) {
        break;
      }
      KeyParser kp = new KeyParser(key,
          ENTITY_ENTRY_PREFIX.length);
      String entityType = kp.getNextString();
      entityTypes.add(entityType);
      byte[] lookupKey = KeyBuilder.newInstance().add(ENTITY_ENTRY_PREFIX)
          .add(entityType).getBytesForLookup();
      if (lookupKey[lookupKey.length - 1] != 0x0) {
        throw new IOException("Found unexpected end byte in lookup key");
      }
      lookupKey[lookupKey.length - 1] = 0x1;
      iterator.seek(lookupKey);
    }
    return entityTypes;
  } catch(DBException e) {
    throw new IOException(e);            	
  } finally {
    IOUtils.cleanup(LOG, iterator);
  }
}
 
Example 22
Source Project: big-c   Source File: LeveldbTimelineStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Finds all keys in the db that have a given prefix and deletes them on
 * the given write batch.
 */
private void deleteKeysWithPrefix(WriteBatch writeBatch, byte[] prefix,
    LeveldbIterator iterator) {
  for (iterator.seek(prefix); iterator.hasNext(); iterator.next()) {
    byte[] key = iterator.peekNext().getKey();
    if (!prefixMatches(prefix, prefix.length, key)) {
      break;
    }
    writeBatch.delete(key);
  }
}
 
Example 23
Source Project: big-c   Source File: LeveldbTimelineStore.java    License: Apache License 2.0 5 votes vote down vote up
private static TimelineDomain getTimelineDomain(
    LeveldbIterator iterator, String domainId, byte[] prefix) throws IOException {
  // Iterate over all the rows whose key starts with prefix to retrieve the
  // domain information.
  TimelineDomain domain = new TimelineDomain();
  domain.setId(domainId);
  boolean noRows = true;
  for (; iterator.hasNext(); iterator.next()) {
    byte[] key = iterator.peekNext().getKey();
    if (!prefixMatches(prefix, prefix.length, key)) {
      break;
    }
    if (noRows) {
      noRows = false;
    }
    byte[] value = iterator.peekNext().getValue();
    if (value != null && value.length > 0) {
      if (key[prefix.length] == DESCRIPTION_COLUMN[0]) {
        domain.setDescription(new String(value, Charset.forName("UTF-8")));
      } else if (key[prefix.length] == OWNER_COLUMN[0]) {
        domain.setOwner(new String(value, Charset.forName("UTF-8")));
      } else if (key[prefix.length] == READER_COLUMN[0]) {
        domain.setReaders(new String(value, Charset.forName("UTF-8")));
      } else if (key[prefix.length] == WRITER_COLUMN[0]) {
        domain.setWriters(new String(value, Charset.forName("UTF-8")));
      } else if (key[prefix.length] == TIMESTAMP_COLUMN[0]) {
        domain.setCreatedTime(readReverseOrderedLong(value, 0));
        domain.setModifiedTime(readReverseOrderedLong(value, 8));
      } else {
        LOG.error("Unrecognized domain column: " + key[prefix.length]);
      }
    }
  }
  if (noRows) {
    return null;
  } else {
    return domain;
  }
}
 
Example 24
Source Project: big-c   Source File: TestLeveldbTimelineStore.java    License: Apache License 2.0 5 votes vote down vote up
private boolean deleteNextEntity(String entityType, byte[] ts)
    throws IOException, InterruptedException {
  LeveldbIterator iterator = null;
  LeveldbIterator pfIterator = null;
  try {
    iterator = ((LeveldbTimelineStore)store).getDbIterator(false);
    pfIterator = ((LeveldbTimelineStore)store).getDbIterator(false);
    return ((LeveldbTimelineStore)store).deleteNextEntity(entityType, ts,
        iterator, pfIterator, false);
  } catch(DBException e) {
    throw new IOException(e);   	
  } finally {
    IOUtils.cleanup(null, iterator, pfIterator);
  }
}
 
Example 25
Source Project: hadoop   Source File: LeveldbTimelineStore.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Read entity from a db iterator.  If no information is found in the
 * specified fields for this entity, return null.
 */
private static TimelineEntity getEntity(String entityId, String entityType,
    Long startTime, EnumSet<Field> fields, LeveldbIterator iterator,
    byte[] prefix, int prefixlen) throws IOException {
  if (fields == null) {
    fields = EnumSet.allOf(Field.class);
  }

  TimelineEntity entity = new TimelineEntity();
  boolean events = false;
  boolean lastEvent = false;
  if (fields.contains(Field.EVENTS)) {
    events = true;
  } else if (fields.contains(Field.LAST_EVENT_ONLY)) {
    lastEvent = true;
  } else {
    entity.setEvents(null);
  }
  boolean relatedEntities = false;
  if (fields.contains(Field.RELATED_ENTITIES)) {
    relatedEntities = true;
  } else {
    entity.setRelatedEntities(null);
  }
  boolean primaryFilters = false;
  if (fields.contains(Field.PRIMARY_FILTERS)) {
    primaryFilters = true;
  } else {
    entity.setPrimaryFilters(null);
  }
  boolean otherInfo = false;
  if (fields.contains(Field.OTHER_INFO)) {
    otherInfo = true;
  } else {
    entity.setOtherInfo(null);
  }

  // iterate through the entity's entry, parsing information if it is part
  // of a requested field
  for (; iterator.hasNext(); iterator.next()) {
    byte[] key = iterator.peekNext().getKey();
    if (!prefixMatches(prefix, prefixlen, key)) {
      break;
    }
    if (key.length == prefixlen) {
      continue;
    }
    if (key[prefixlen] == PRIMARY_FILTERS_COLUMN[0]) {
      if (primaryFilters) {
        addPrimaryFilter(entity, key,
            prefixlen + PRIMARY_FILTERS_COLUMN.length);
      }
    } else if (key[prefixlen] == OTHER_INFO_COLUMN[0]) {
      if (otherInfo) {
        entity.addOtherInfo(parseRemainingKey(key,
            prefixlen + OTHER_INFO_COLUMN.length),
            GenericObjectMapper.read(iterator.peekNext().getValue()));
      }
    } else if (key[prefixlen] == RELATED_ENTITIES_COLUMN[0]) {
      if (relatedEntities) {
        addRelatedEntity(entity, key,
            prefixlen + RELATED_ENTITIES_COLUMN.length);
      }
    } else if (key[prefixlen] == EVENTS_COLUMN[0]) {
      if (events || (lastEvent &&
          entity.getEvents().size() == 0)) {
        TimelineEvent event = getEntityEvent(null, key, prefixlen +
            EVENTS_COLUMN.length, iterator.peekNext().getValue());
        if (event != null) {
          entity.addEvent(event);
        }
      }
    } else if (key[prefixlen] == DOMAIN_ID_COLUMN[0]) {
      byte[] v = iterator.peekNext().getValue();
      String domainId = new String(v, Charset.forName("UTF-8"));
      entity.setDomainId(domainId);
    } else {
      if (key[prefixlen] !=
          INVISIBLE_REVERSE_RELATED_ENTITIES_COLUMN[0]) {
        LOG.warn(String.format("Found unexpected column for entity %s of " +
            "type %s (0x%02x)", entityId, entityType, key[prefixlen]));
      }
    }
  }

  entity.setEntityId(entityId);
  entity.setEntityType(entityType);
  entity.setStartTime(startTime);

  return entity;
}
 
Example 26
Source Project: hadoop   Source File: LeveldbTimelineStore.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Discards entities with start timestamp less than or equal to the given
 * timestamp.
 */
@VisibleForTesting
void discardOldEntities(long timestamp)
    throws IOException, InterruptedException {
  byte[] reverseTimestamp = writeReverseOrderedLong(timestamp);
  long totalCount = 0;
  long t1 = System.currentTimeMillis();
  try {
    List<String> entityTypes = getEntityTypes();
    for (String entityType : entityTypes) {
      LeveldbIterator iterator = null;
      LeveldbIterator pfIterator = null;
      long typeCount = 0;
      try {
        deleteLock.writeLock().lock();
        iterator = getDbIterator(false);
        pfIterator = getDbIterator(false);

        if (deletionThread != null && deletionThread.isInterrupted()) {
          throw new InterruptedException();
        }
        boolean seeked = false;
        while (deleteNextEntity(entityType, reverseTimestamp, iterator,
            pfIterator, seeked)) {
          typeCount++;
          totalCount++;
          seeked = true;
          if (deletionThread != null && deletionThread.isInterrupted()) {
            throw new InterruptedException();
          }
        }
      } catch (IOException e) {
        LOG.error("Got IOException while deleting entities for type " +
            entityType + ", continuing to next type", e);
      } finally {
        IOUtils.cleanup(LOG, iterator, pfIterator);
        deleteLock.writeLock().unlock();
        if (typeCount > 0) {
          LOG.info("Deleted " + typeCount + " entities of type " +
              entityType);
        }
      }
    }
  } finally {
    long t2 = System.currentTimeMillis();
    LOG.info("Discarded " + totalCount + " entities for timestamp " +
        timestamp + " and earlier in " + (t2 - t1) / 1000.0 + " seconds");
  }
}
 
Example 27
Source Project: hadoop   Source File: LeveldbTimelineStore.java    License: Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
LeveldbIterator getDbIterator(boolean fillCache) {
  ReadOptions readOptions = new ReadOptions();
  readOptions.fillCache(fillCache);
  return new LeveldbIterator(db, readOptions);
}
 
Example 28
Source Project: hadoop   Source File: LeveldbTimelineStore.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TimelineDomains getDomains(String owner)
    throws IOException {
  LeveldbIterator iterator = null;
  try {
    byte[] prefix = KeyBuilder.newInstance()
        .add(OWNER_LOOKUP_PREFIX).add(owner).getBytesForLookup();
    List<TimelineDomain> domains = new ArrayList<TimelineDomain>();
    for (iterator = new LeveldbIterator(db), iterator.seek(prefix);
        iterator.hasNext();) {
      byte[] key = iterator.peekNext().getKey();
      if (!prefixMatches(prefix, prefix.length, key)) {
        break;
      }
      // Iterator to parse the rows of an individual domain
      KeyParser kp = new KeyParser(key, prefix.length);
      String domainId = kp.getNextString();
      byte[] prefixExt = KeyBuilder.newInstance().add(OWNER_LOOKUP_PREFIX)
          .add(owner).add(domainId).getBytesForLookup();
      TimelineDomain domainToReturn =
          getTimelineDomain(iterator, domainId, prefixExt);
      if (domainToReturn != null) {
        domains.add(domainToReturn);
      }
    }
    // Sort the domains to return
    Collections.sort(domains, new Comparator<TimelineDomain>() {
      @Override
      public int compare(
          TimelineDomain domain1, TimelineDomain domain2) {
         int result = domain2.getCreatedTime().compareTo(
             domain1.getCreatedTime());
         if (result == 0) {
           return domain2.getModifiedTime().compareTo(
               domain1.getModifiedTime());
         } else {
           return result;
         }
      }
    });
    TimelineDomains domainsToReturn = new TimelineDomains();
    domainsToReturn.addDomains(domains);
    return domainsToReturn;
  } catch(DBException e) {
    throw new IOException(e);            	
  } finally {
    IOUtils.cleanup(LOG, iterator);
  }
}
 
Example 29
Source Project: big-c   Source File: LeveldbTimelineStore.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Read entity from a db iterator.  If no information is found in the
 * specified fields for this entity, return null.
 */
private static TimelineEntity getEntity(String entityId, String entityType,
    Long startTime, EnumSet<Field> fields, LeveldbIterator iterator,
    byte[] prefix, int prefixlen) throws IOException {
  if (fields == null) {
    fields = EnumSet.allOf(Field.class);
  }

  TimelineEntity entity = new TimelineEntity();
  boolean events = false;
  boolean lastEvent = false;
  if (fields.contains(Field.EVENTS)) {
    events = true;
  } else if (fields.contains(Field.LAST_EVENT_ONLY)) {
    lastEvent = true;
  } else {
    entity.setEvents(null);
  }
  boolean relatedEntities = false;
  if (fields.contains(Field.RELATED_ENTITIES)) {
    relatedEntities = true;
  } else {
    entity.setRelatedEntities(null);
  }
  boolean primaryFilters = false;
  if (fields.contains(Field.PRIMARY_FILTERS)) {
    primaryFilters = true;
  } else {
    entity.setPrimaryFilters(null);
  }
  boolean otherInfo = false;
  if (fields.contains(Field.OTHER_INFO)) {
    otherInfo = true;
  } else {
    entity.setOtherInfo(null);
  }

  // iterate through the entity's entry, parsing information if it is part
  // of a requested field
  for (; iterator.hasNext(); iterator.next()) {
    byte[] key = iterator.peekNext().getKey();
    if (!prefixMatches(prefix, prefixlen, key)) {
      break;
    }
    if (key.length == prefixlen) {
      continue;
    }
    if (key[prefixlen] == PRIMARY_FILTERS_COLUMN[0]) {
      if (primaryFilters) {
        addPrimaryFilter(entity, key,
            prefixlen + PRIMARY_FILTERS_COLUMN.length);
      }
    } else if (key[prefixlen] == OTHER_INFO_COLUMN[0]) {
      if (otherInfo) {
        entity.addOtherInfo(parseRemainingKey(key,
            prefixlen + OTHER_INFO_COLUMN.length),
            GenericObjectMapper.read(iterator.peekNext().getValue()));
      }
    } else if (key[prefixlen] == RELATED_ENTITIES_COLUMN[0]) {
      if (relatedEntities) {
        addRelatedEntity(entity, key,
            prefixlen + RELATED_ENTITIES_COLUMN.length);
      }
    } else if (key[prefixlen] == EVENTS_COLUMN[0]) {
      if (events || (lastEvent &&
          entity.getEvents().size() == 0)) {
        TimelineEvent event = getEntityEvent(null, key, prefixlen +
            EVENTS_COLUMN.length, iterator.peekNext().getValue());
        if (event != null) {
          entity.addEvent(event);
        }
      }
    } else if (key[prefixlen] == DOMAIN_ID_COLUMN[0]) {
      byte[] v = iterator.peekNext().getValue();
      String domainId = new String(v, Charset.forName("UTF-8"));
      entity.setDomainId(domainId);
    } else {
      if (key[prefixlen] !=
          INVISIBLE_REVERSE_RELATED_ENTITIES_COLUMN[0]) {
        LOG.warn(String.format("Found unexpected column for entity %s of " +
            "type %s (0x%02x)", entityId, entityType, key[prefixlen]));
      }
    }
  }

  entity.setEntityId(entityId);
  entity.setEntityType(entityType);
  entity.setStartTime(startTime);

  return entity;
}
 
Example 30
Source Project: big-c   Source File: LeveldbTimelineStore.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Discards entities with start timestamp less than or equal to the given
 * timestamp.
 */
@VisibleForTesting
void discardOldEntities(long timestamp)
    throws IOException, InterruptedException {
  byte[] reverseTimestamp = writeReverseOrderedLong(timestamp);
  long totalCount = 0;
  long t1 = System.currentTimeMillis();
  try {
    List<String> entityTypes = getEntityTypes();
    for (String entityType : entityTypes) {
      LeveldbIterator iterator = null;
      LeveldbIterator pfIterator = null;
      long typeCount = 0;
      try {
        deleteLock.writeLock().lock();
        iterator = getDbIterator(false);
        pfIterator = getDbIterator(false);

        if (deletionThread != null && deletionThread.isInterrupted()) {
          throw new InterruptedException();
        }
        boolean seeked = false;
        while (deleteNextEntity(entityType, reverseTimestamp, iterator,
            pfIterator, seeked)) {
          typeCount++;
          totalCount++;
          seeked = true;
          if (deletionThread != null && deletionThread.isInterrupted()) {
            throw new InterruptedException();
          }
        }
      } catch (IOException e) {
        LOG.error("Got IOException while deleting entities for type " +
            entityType + ", continuing to next type", e);
      } finally {
        IOUtils.cleanup(LOG, iterator, pfIterator);
        deleteLock.writeLock().unlock();
        if (typeCount > 0) {
          LOG.info("Deleted " + typeCount + " entities of type " +
              entityType);
        }
      }
    }
  } finally {
    long t2 = System.currentTimeMillis();
    LOG.info("Discarded " + totalCount + " entities for timestamp " +
        timestamp + " and earlier in " + (t2 - t1) / 1000.0 + " seconds");
  }
}