Java Code Examples for org.apache.hadoop.classification.InterfaceAudience.Private

The following examples show how to use org.apache.hadoop.classification.InterfaceAudience.Private. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: ZKRMStateStore.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Given the {@link Configuration} and {@link ACL}s used (zkAcl) for
 * ZooKeeper access, construct the {@link ACL}s for the store's root node.
 * In the constructed {@link ACL}, all the users allowed by zkAcl are given
 * rwa access, while the current RM has exclude create-delete access.
 *
 * To be called only when HA is enabled and the configuration doesn't set ACL
 * for the root node.
 */
@VisibleForTesting
@Private
@Unstable
protected List<ACL> constructZkRootNodeACL(
    Configuration conf, List<ACL> sourceACLs) throws NoSuchAlgorithmException {
  List<ACL> zkRootNodeAcl = new ArrayList<ACL>();
  for (ACL acl : sourceACLs) {
    zkRootNodeAcl.add(new ACL(
        ZKUtil.removeSpecificPerms(acl.getPerms(), CREATE_DELETE_PERMS),
        acl.getId()));
  }

  zkRootNodeUsername = HAUtil.getConfValueForRMInstance(
      YarnConfiguration.RM_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADDRESS, conf);
  Id rmId = new Id(zkRootNodeAuthScheme,
      DigestAuthenticationProvider.generateDigest(
          zkRootNodeUsername + ":" + zkRootNodePassword));
  zkRootNodeAcl.add(new ACL(CREATE_DELETE_PERMS, rmId));
  return zkRootNodeAcl;
}
 
Example 2
Source Project: hadoop   Source File: SharedCacheUtil.java    License: Apache License 2.0 6 votes vote down vote up
@Private
public static String getCacheEntryPath(int cacheDepth, String cacheRoot,
    String checksum) {

  if (cacheDepth <= 0) {
    throw new IllegalArgumentException(
        "The cache depth must be greater than 0. Passed value: " + cacheDepth);
  }
  if (checksum.length() < cacheDepth) {
    throw new IllegalArgumentException("The checksum passed was too short: "
        + checksum);
  }

  // Build the cache entry path to the specified depth. For example, if the
  // depth is 3 and the checksum is 3c4f, the path would be:
  // SHARED_CACHE_ROOT/3/c/4/3c4f
  StringBuilder sb = new StringBuilder(cacheRoot);
  for (int i = 0; i < cacheDepth; i++) {
    sb.append(Path.SEPARATOR_CHAR);
    sb.append(checksum.charAt(i));
  }
  sb.append(Path.SEPARATOR_CHAR).append(checksum);

  return sb.toString();
}
 
Example 3
Source Project: hadoop   Source File: SCMStore.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Clean all resource references to a cache resource that contain application
 * ids pointing to finished applications. If the resource key does not exist,
 * do nothing.
 *
 * @param key a unique identifier for a resource
 * @throws YarnException
 */
@Private
public void cleanResourceReferences(String key) throws YarnException {
  Collection<SharedCacheResourceReference> refs = getResourceReferences(key);
  if (!refs.isEmpty()) {
    Set<SharedCacheResourceReference> refsToRemove =
        new HashSet<SharedCacheResourceReference>();
    for (SharedCacheResourceReference r : refs) {
      if (!appChecker.isApplicationActive(r.getAppId())) {
        // application in resource reference is dead, it is safe to remove the
        // reference
        refsToRemove.add(r);
      }
    }
    if (refsToRemove.size() > 0) {
      removeResourceReferences(key, refsToRemove, false);
    }
  }
}
 
Example 4
Source Project: hadoop   Source File: SCMStore.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create an instance of the AppChecker service via reflection based on the
 * {@link YarnConfiguration#SCM_APP_CHECKER_CLASS} parameter.
 * 
 * @param conf
 * @return an instance of the AppChecker class
 */
@Private
@SuppressWarnings("unchecked")
public static AppChecker createAppCheckerService(Configuration conf) {
  Class<? extends AppChecker> defaultCheckerClass;
  try {
    defaultCheckerClass =
        (Class<? extends AppChecker>) Class
            .forName(YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS);
  } catch (Exception e) {
    throw new YarnRuntimeException("Invalid default scm app checker class"
        + YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS, e);
  }

  AppChecker checker =
      ReflectionUtils.newInstance(conf.getClass(
          YarnConfiguration.SCM_APP_CHECKER_CLASS, defaultCheckerClass,
          AppChecker.class), conf);
  return checker;
}
 
Example 5
Source Project: hadoop   Source File: AMRMTokenIdentifier.java    License: Apache License 2.0 5 votes vote down vote up
@Private
public ApplicationAttemptId getApplicationAttemptId() {
  if (!proto.hasAppAttemptId()) {
    return null;
  }
  return new ApplicationAttemptIdPBImpl(proto.getAppAttemptId());
}
 
Example 6
Source Project: hadoop   Source File: ContainerContext.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public ContainerContext(String user, ContainerId containerId,
    Resource resource) {
  this.user = user;
  this.containerId = containerId;
  this.resource = resource;
}
 
Example 7
Source Project: hadoop   Source File: ApplicationReport.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static ApplicationReport newInstance(ApplicationId applicationId,
    ApplicationAttemptId applicationAttemptId, String user, String queue,
    String name, String host, int rpcPort, Token clientToAMToken,
    YarnApplicationState state, String diagnostics, String url,
    long startTime, long finishTime, FinalApplicationStatus finalStatus,
    ApplicationResourceUsageReport appResources, String origTrackingUrl,
    float progress, String applicationType, Token amRmToken) {
  ApplicationReport report = Records.newRecord(ApplicationReport.class);
  report.setApplicationId(applicationId);
  report.setCurrentApplicationAttemptId(applicationAttemptId);
  report.setUser(user);
  report.setQueue(queue);
  report.setName(name);
  report.setHost(host);
  report.setRpcPort(rpcPort);
  report.setClientToAMToken(clientToAMToken);
  report.setYarnApplicationState(state);
  report.setDiagnostics(diagnostics);
  report.setTrackingUrl(url);
  report.setStartTime(startTime);
  report.setFinishTime(finishTime);
  report.setFinalApplicationStatus(finalStatus);
  report.setApplicationResourceUsageReport(appResources);
  report.setOriginalTrackingUrl(origTrackingUrl);
  report.setProgress(progress);
  report.setApplicationType(applicationType);
  report.setAMRMToken(amRmToken);
  return report;
}
 
Example 8
Source Project: hadoop   Source File: ApplicationId.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static ApplicationId newInstance(long clusterTimestamp, int id) {
  ApplicationId appId = Records.newRecord(ApplicationId.class);
  appId.setClusterTimestamp(clusterTimestamp);
  appId.setId(id);
  appId.build();
  return appId;
}
 
Example 9
Source Project: hadoop   Source File: RMProxy.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get the ResourceManager address from the provided Configuration for the
 * given protocol.
 */
@Private
protected InetSocketAddress getRMAddress(
    YarnConfiguration conf, Class<?> protocol) throws IOException {
  throw new UnsupportedOperationException("This method should be invoked " +
      "from an instance of ClientRMProxy or ServerRMProxy");
}
 
Example 10
Source Project: hadoop   Source File: RMProxy.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get a proxy to the RM at the specified address. To be used to create a
 * RetryProxy.
 */
@Private
static <T> T getProxy(final Configuration conf,
    final Class<T> protocol, final InetSocketAddress rmAddress)
    throws IOException {
  return UserGroupInformation.getCurrentUser().doAs(
    new PrivilegedAction<T>() {
      @Override
      public T run() {
        return (T) YarnRPC.create(conf).getProxy(protocol, rmAddress, conf);
      }
    });
}
 
Example 11
Source Project: hadoop   Source File: ContainerStatus.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static ContainerStatus newInstance(ContainerId containerId,
    ContainerState containerState, String diagnostics, int exitStatus) {
  ContainerStatus containerStatus = Records.newRecord(ContainerStatus.class);
  containerStatus.setState(containerState);
  containerStatus.setContainerId(containerId);
  containerStatus.setDiagnostics(diagnostics);
  containerStatus.setExitStatus(exitStatus);
  return containerStatus;
}
 
Example 12
Source Project: hadoop   Source File: ClientRMProxy.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Override
protected void checkAllowedProtocols(Class<?> protocol) {
  Preconditions.checkArgument(
      protocol.isAssignableFrom(ClientRMProtocols.class),
      "RM does not support this client protocol");
}
 
Example 13
Source Project: hadoop   Source File: RefreshNodesResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static RefreshNodesResponse newInstance() {
  RefreshNodesResponse response =
      Records.newRecord(RefreshNodesResponse.class);
  return response;
}
 
Example 14
Source Project: hadoop   Source File: AMRMClientImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@VisibleForTesting
protected void populateNMTokens(List<NMToken> nmTokens) {
  for (NMToken token : nmTokens) {
    String nodeId = token.getNodeId().toString();
    if (getNMTokenCache().containsToken(nodeId)) {
      LOG.info("Replacing token for : " + nodeId);
    } else {
      LOG.info("Received new token for : " + nodeId);
    }
    getNMTokenCache().setToken(nodeId, token.getToken());
  }
}
 
Example 15
Source Project: hadoop   Source File: GetNewApplicationResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static GetNewApplicationResponse newInstance(
    ApplicationId applicationId, Resource minCapability,
    Resource maxCapability) {
  GetNewApplicationResponse response =
      Records.newRecord(GetNewApplicationResponse.class);
  response.setApplicationId(applicationId);
  response.setMaximumResourceCapability(maxCapability);
  return response;
}
 
Example 16
Source Project: hadoop   Source File: CancelDelegationTokenResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static CancelDelegationTokenResponse newInstance() {
  CancelDelegationTokenResponse response =
      Records.newRecord(CancelDelegationTokenResponse.class);
  return response;
}
 
Example 17
Source Project: hadoop   Source File: GetQueueInfoResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static GetQueueInfoResponse newInstance(QueueInfo queueInfo) {
  GetQueueInfoResponse response = Records.newRecord(GetQueueInfoResponse.class);
  response.setQueueInfo(queueInfo);
  return response;
}
 
Example 18
Source Project: hadoop   Source File: GetDelegationTokenResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static GetDelegationTokenResponse newInstance(Token rmDTToken) {
  GetDelegationTokenResponse response =
      Records.newRecord(GetDelegationTokenResponse.class);
  response.setRMDelegationToken(rmDTToken);
  return response;
}
 
Example 19
Source Project: hadoop   Source File: StopContainersResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static StopContainersResponse newInstance(
    List<ContainerId> succeededRequests,
    Map<ContainerId, SerializedException> failedRequests) {
  StopContainersResponse response =
      Records.newRecord(StopContainersResponse.class);
  response.setFailedRequests(failedRequests);
  response.setSuccessfullyStoppedContainers(succeededRequests);
  return response;
}
 
Example 20
Source Project: hadoop   Source File: GetApplicationReportResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static GetApplicationReportResponse newInstance(
    ApplicationReport ApplicationReport) {
  GetApplicationReportResponse response =
      Records.newRecord(GetApplicationReportResponse.class);
  response.setApplicationReport(ApplicationReport);
  return response;
}
 
Example 21
Source Project: hadoop   Source File: YarnClusterMetrics.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static YarnClusterMetrics newInstance(int numNodeManagers) {
  YarnClusterMetrics metrics = Records.newRecord(YarnClusterMetrics.class);
  metrics.setNumNodeManagers(numNodeManagers);
  return metrics;
}
 
Example 22
Source Project: hadoop   Source File: HostsFileReader.java    License: Apache License 2.0 5 votes vote down vote up
@Private
public HostsFileReader(String includesFile, InputStream inFileInputStream,
    String excludesFile, InputStream exFileInputStream) throws IOException {
  includes = new HashSet<String>();
  excludes = new HashSet<String>();
  this.includesFile = includesFile;
  this.excludesFile = excludesFile;
  refresh(inFileInputStream, exFileInputStream);
}
 
Example 23
Source Project: dremio-oss   Source File: HadoopFileSystemWrapper.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Private
public Token<?> getDelegationToken(String renewer) throws IOException {
  try (WaitRecorder recorder = OperatorStats.getWaitRecorder(operatorStats)) {
    return underlyingFs.getDelegationToken(renewer);
  } catch(FSError e) {
    throw propagateFSError(e);
  }
}
 
Example 24
Source Project: hadoop   Source File: SubmitApplicationResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static SubmitApplicationResponse newInstance() {
  SubmitApplicationResponse response =
      Records.newRecord(SubmitApplicationResponse.class);
  return response;
}
 
Example 25
Source Project: hadoop   Source File: RMContainerAllocator.java    License: Apache License 2.0 5 votes vote down vote up
@Private
public Resource getResourceLimit() {
  Resource headRoom = getAvailableResources();
  if (headRoom == null) {
    headRoom = Resources.none();
  }
  Resource assignedMapResource =
      Resources.multiply(mapResourceRequest, assignedRequests.maps.size());
  Resource assignedReduceResource =
      Resources.multiply(reduceResourceRequest,
        assignedRequests.reduces.size());
  return Resources.add(headRoom,
    Resources.add(assignedMapResource, assignedReduceResource));
}
 
Example 26
Source Project: hadoop   Source File: ReservationUpdateResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static ReservationUpdateResponse newInstance() {
  ReservationUpdateResponse response =
      Records.newRecord(ReservationUpdateResponse.class);
  return response;
}
 
Example 27
@Private
@Unstable
public static RefreshUserToGroupsMappingsResponse newInstance() {
  RefreshUserToGroupsMappingsResponse response =
      Records.newRecord(RefreshUserToGroupsMappingsResponse.class);
  return response;
}
 
Example 28
Source Project: hadoop   Source File: AMRMClientAsync.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@VisibleForTesting
protected AMRMClientAsync(AMRMClient<T> client, int intervalMs,
    CallbackHandler callbackHandler) {
  super(AMRMClientAsync.class.getName());
  this.client = client;
  this.heartbeatIntervalMs.set(intervalMs);
  this.handler = callbackHandler;
}
 
Example 29
Source Project: hadoop   Source File: AMRMTokenSecretManager.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@VisibleForTesting
public MasterKeyData getNextMasterKeyData() {
  this.readLock.lock();
  try {
    return this.nextMasterKey;
  } finally {
    this.readLock.unlock();
  }
}
 
Example 30
Source Project: hadoop   Source File: RMActiveServiceContext.java    License: Apache License 2.0 4 votes vote down vote up
@Private
@Unstable
public void setRMDelegationTokenSecretManager(
    RMDelegationTokenSecretManager delegationTokenSecretManager) {
  this.rmDelegationTokenSecretManager = delegationTokenSecretManager;
}