Java Code Examples for org.apache.hadoop.yarn.server.utils.LeveldbIterator#peekNext()

The following examples show how to use org.apache.hadoop.yarn.server.utils.LeveldbIterator#peekNext() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NMLeveldbStateStoreService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private LocalResourceTrackerState loadResourceTrackerState(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  final String completedPrefix = keyPrefix + LOCALIZATION_COMPLETED_SUFFIX;
  final String startedPrefix = keyPrefix + LOCALIZATION_STARTED_SUFFIX;
  LocalResourceTrackerState state = new LocalResourceTrackerState();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (key.startsWith(completedPrefix)) {
      state.localizedResources = loadCompletedResources(iter,
          completedPrefix);
    } else if (key.startsWith(startedPrefix)) {
      state.inProgressResources = loadStartedResources(iter, startedPrefix);
    } else {
      throw new IOException("Unexpected key in resource tracker state: "
          + key);
    }
  }

  return state;
}
Example 2
Source File: NMLeveldbStateStoreService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private List<LocalizedResourceProto> loadCompletedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  List<LocalizedResourceProto> rsrcs =
      new ArrayList<LocalizedResourceProto>();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("Loading completed resource from " + key);
    }
    rsrcs.add(LocalizedResourceProto.parseFrom(entry.getValue()));
    iter.next();
  }

  return rsrcs;
}
Example 3
Source File: NMLeveldbStateStoreService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private Map<LocalResourceProto, Path> loadStartedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  Map<LocalResourceProto, Path> rsrcs =
      new HashMap<LocalResourceProto, Path>();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    Path localPath = new Path(key.substring(keyPrefix.length()));
    if (LOG.isDebugEnabled()) {
      LOG.debug("Loading in-progress resource at " + localPath);
    }
    rsrcs.put(LocalResourceProto.parseFrom(entry.getValue()), localPath);
    iter.next();
  }

  return rsrcs;
}
Example 4
Source File: NMLeveldbStateStoreService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private LocalResourceTrackerState loadResourceTrackerState(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  final String completedPrefix = keyPrefix + LOCALIZATION_COMPLETED_SUFFIX;
  final String startedPrefix = keyPrefix + LOCALIZATION_STARTED_SUFFIX;
  LocalResourceTrackerState state = new LocalResourceTrackerState();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (key.startsWith(completedPrefix)) {
      state.localizedResources = loadCompletedResources(iter,
          completedPrefix);
    } else if (key.startsWith(startedPrefix)) {
      state.inProgressResources = loadStartedResources(iter, startedPrefix);
    } else {
      throw new IOException("Unexpected key in resource tracker state: "
          + key);
    }
  }

  return state;
}
Example 5
Source File: NMLeveldbStateStoreService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private List<LocalizedResourceProto> loadCompletedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  List<LocalizedResourceProto> rsrcs =
      new ArrayList<LocalizedResourceProto>();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("Loading completed resource from " + key);
    }
    rsrcs.add(LocalizedResourceProto.parseFrom(entry.getValue()));
    iter.next();
  }

  return rsrcs;
}
Example 6
Source File: NMLeveldbStateStoreService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private Map<LocalResourceProto, Path> loadStartedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  Map<LocalResourceProto, Path> rsrcs =
      new HashMap<LocalResourceProto, Path>();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    Path localPath = new Path(key.substring(keyPrefix.length()));
    if (LOG.isDebugEnabled()) {
      LOG.debug("Loading in-progress resource at " + localPath);
    }
    rsrcs.put(LocalResourceProto.parseFrom(entry.getValue()), localPath);
    iter.next();
  }

  return rsrcs;
}
Example 7
Source File: NMLeveldbStateStoreService.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private RecoveredContainerState loadContainerState(ContainerId containerId,
    LeveldbIterator iter, String keyPrefix) throws IOException {
  RecoveredContainerState rcs = new RecoveredContainerState();
  rcs.status = RecoveredContainerStatus.REQUESTED;
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }
    iter.next();

    String suffix = key.substring(keyPrefix.length()-1);  // start with '/'
    if (suffix.equals(CONTAINER_REQUEST_KEY_SUFFIX)) {
      rcs.startRequest = new StartContainerRequestPBImpl(
          StartContainerRequestProto.parseFrom(entry.getValue()));
    } else if (suffix.equals(CONTAINER_DIAGS_KEY_SUFFIX)) {
      rcs.diagnostics = asString(entry.getValue());
    } else if (suffix.equals(CONTAINER_LAUNCHED_KEY_SUFFIX)) {
      if (rcs.status == RecoveredContainerStatus.REQUESTED) {
        rcs.status = RecoveredContainerStatus.LAUNCHED;
      }
    } else if (suffix.equals(CONTAINER_KILLED_KEY_SUFFIX)) {
      rcs.killed = true;
    } else if (suffix.equals(CONTAINER_EXIT_CODE_KEY_SUFFIX)) {
      rcs.status = RecoveredContainerStatus.COMPLETED;
      rcs.exitCode = Integer.parseInt(asString(entry.getValue()));
    } else {
      throw new IOException("Unexpected container state key: " + key);
    }
  }
  return rcs;
}
Example 8
Source File: NMLeveldbStateStoreService.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private RecoveredUserResources loadUserLocalizedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  RecoveredUserResources userResources = new RecoveredUserResources();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (key.startsWith(LOCALIZATION_FILECACHE_SUFFIX, keyPrefix.length())) {
      userResources.privateTrackerState = loadResourceTrackerState(iter,
          keyPrefix + LOCALIZATION_FILECACHE_SUFFIX);
    } else if (key.startsWith(LOCALIZATION_APPCACHE_SUFFIX,
        keyPrefix.length())) {
      int appIdStartPos = keyPrefix.length() +
          LOCALIZATION_APPCACHE_SUFFIX.length();
      int appIdEndPos = key.indexOf('/', appIdStartPos);
      if (appIdEndPos < 0) {
        throw new IOException("Unable to determine appID in resource key: "
            + key);
      }
      ApplicationId appId = ConverterUtils.toApplicationId(
          key.substring(appIdStartPos, appIdEndPos));
      userResources.appTrackerStates.put(appId,
          loadResourceTrackerState(iter, key.substring(0, appIdEndPos+1)));
    } else {
      throw new IOException("Unexpected user resource key " + key);
    }
  }
  return userResources;
}
Example 9
Source File: LeveldbRMStateStore.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private int loadRMApp(RMState rmState, LeveldbIterator iter, String appIdStr,
    byte[] appData) throws IOException {
  ApplicationStateData appState = createApplicationState(appIdStr, appData);
  ApplicationId appId =
      appState.getApplicationSubmissionContext().getApplicationId();
  rmState.appState.put(appId, appState);
  String attemptNodePrefix = getApplicationNodeKey(appId) + SEPARATOR;
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(attemptNodePrefix)) {
      break;
    }

    String attemptId = key.substring(attemptNodePrefix.length());
    if (attemptId.startsWith(ApplicationAttemptId.appAttemptIdStrPrefix)) {
      ApplicationAttemptStateData attemptState =
          createAttemptState(attemptId, entry.getValue());
      appState.attempts.put(attemptState.getAttemptId(), attemptState);
    } else {
      LOG.warn("Ignoring unknown application key: " + key);
    }
    iter.next();
  }
  int numAttempts = appState.attempts.size();
  if (LOG.isDebugEnabled()) {
    LOG.debug("Loaded application " + appId + " with " + numAttempts
        + " attempts");
  }
  return numAttempts;
}
Example 10
Source File: NMLeveldbStateStoreService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private RecoveredContainerState loadContainerState(ContainerId containerId,
    LeveldbIterator iter, String keyPrefix) throws IOException {
  RecoveredContainerState rcs = new RecoveredContainerState();
  rcs.status = RecoveredContainerStatus.REQUESTED;
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }
    iter.next();

    String suffix = key.substring(keyPrefix.length()-1);  // start with '/'
    if (suffix.equals(CONTAINER_REQUEST_KEY_SUFFIX)) {
      rcs.startRequest = new StartContainerRequestPBImpl(
          StartContainerRequestProto.parseFrom(entry.getValue()));
    } else if (suffix.equals(CONTAINER_DIAGS_KEY_SUFFIX)) {
      rcs.diagnostics = asString(entry.getValue());
    } else if (suffix.equals(CONTAINER_LAUNCHED_KEY_SUFFIX)) {
      if (rcs.status == RecoveredContainerStatus.REQUESTED) {
        rcs.status = RecoveredContainerStatus.LAUNCHED;
      }
    } else if (suffix.equals(CONTAINER_KILLED_KEY_SUFFIX)) {
      rcs.killed = true;
    } else if (suffix.equals(CONTAINER_EXIT_CODE_KEY_SUFFIX)) {
      rcs.status = RecoveredContainerStatus.COMPLETED;
      rcs.exitCode = Integer.parseInt(asString(entry.getValue()));
    } else {
      throw new IOException("Unexpected container state key: " + key);
    }
  }
  return rcs;
}
Example 11
Source File: NMLeveldbStateStoreService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private RecoveredUserResources loadUserLocalizedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  RecoveredUserResources userResources = new RecoveredUserResources();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (key.startsWith(LOCALIZATION_FILECACHE_SUFFIX, keyPrefix.length())) {
      userResources.privateTrackerState = loadResourceTrackerState(iter,
          keyPrefix + LOCALIZATION_FILECACHE_SUFFIX);
    } else if (key.startsWith(LOCALIZATION_APPCACHE_SUFFIX,
        keyPrefix.length())) {
      int appIdStartPos = keyPrefix.length() +
          LOCALIZATION_APPCACHE_SUFFIX.length();
      int appIdEndPos = key.indexOf('/', appIdStartPos);
      if (appIdEndPos < 0) {
        throw new IOException("Unable to determine appID in resource key: "
            + key);
      }
      ApplicationId appId = ConverterUtils.toApplicationId(
          key.substring(appIdStartPos, appIdEndPos));
      userResources.appTrackerStates.put(appId,
          loadResourceTrackerState(iter, key.substring(0, appIdEndPos+1)));
    } else {
      throw new IOException("Unexpected user resource key " + key);
    }
  }
  return userResources;
}
Example 12
Source File: LeveldbRMStateStore.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private int loadRMApp(RMState rmState, LeveldbIterator iter, String appIdStr,
    byte[] appData) throws IOException {
  ApplicationStateData appState = createApplicationState(appIdStr, appData);
  ApplicationId appId =
      appState.getApplicationSubmissionContext().getApplicationId();
  rmState.appState.put(appId, appState);
  String attemptNodePrefix = getApplicationNodeKey(appId) + SEPARATOR;
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(attemptNodePrefix)) {
      break;
    }

    String attemptId = key.substring(attemptNodePrefix.length());
    if (attemptId.startsWith(ApplicationAttemptId.appAttemptIdStrPrefix)) {
      ApplicationAttemptStateData attemptState =
          createAttemptState(attemptId, entry.getValue());
      appState.attempts.put(attemptState.getAttemptId(), attemptState);
    } else {
      LOG.warn("Ignoring unknown application key: " + key);
    }
    iter.next();
  }
  int numAttempts = appState.attempts.size();
  if (LOG.isDebugEnabled()) {
    LOG.debug("Loaded application " + appId + " with " + numAttempts
        + " attempts");
  }
  return numAttempts;
}