Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationId#toString()

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationId#toString() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: TensorFlowOnYARN   File: LaunchCluster.java    License: Apache License 2.0 6 votes vote down vote up
public boolean run() throws Exception {
  YarnClientApplication app = createApplication();
  ApplicationId appId = app.getNewApplicationResponse().getApplicationId();

  // Copy the application jar to the filesystem
  FileSystem fs = FileSystem.get(conf);
  String appIdStr = appId.toString();
  Path dstJarPath = Utils.copyLocalFileToDfs(fs, appIdStr, new Path(tfJar), Constants.TF_JAR_NAME);
  Path dstLibPath = Utils.copyLocalFileToDfs(fs, appIdStr, new Path(tfLib),
      Constants.TF_LIB_NAME);
  Map<String, Path> files = new HashMap<>();
  files.put(Constants.TF_JAR_NAME, dstJarPath);
  Map<String, LocalResource> localResources = Utils.makeLocalResources(fs, files);
  Map<String, String> javaEnv = Utils.setJavaEnv(conf);
  String command = makeAppMasterCommand(dstLibPath.toString(), dstJarPath.toString());
  LOG.info("Make ApplicationMaster command: " + command);
  ContainerLaunchContext launchContext = ContainerLaunchContext.newInstance(
      localResources, javaEnv, Lists.newArrayList(command), null, null, null);
  Resource resource = Resource.newInstance(amMemory, amVCores);
  submitApplication(app, appName, launchContext, resource, amQueue);
  return awaitApplication(appId);
}
 
Example 2
Source Project: flink   File: YARNFileReplicationITCase.java    License: Apache License 2.0 6 votes vote down vote up
private void extraVerification(Configuration configuration, ApplicationId applicationId) throws Exception {
	final FileSystem fs = FileSystem.get(getYarnConfiguration());

	String suffix = ".flink/" + applicationId.toString() + "/" + flinkUberjar.getName();

	Path uberJarHDFSPath = new Path(fs.getHomeDirectory(), suffix);
	FileStatus fsStatus = fs.getFileStatus(uberJarHDFSPath);

	final int flinkFileReplication = configuration.getInteger(YarnConfigOptions.FILE_REPLICATION);
	final int replication = YARN_CONFIGURATION.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, DFSConfigKeys.DFS_REPLICATION_DEFAULT);

	// If YarnConfigOptions.FILE_REPLICATION is not set. The replication number should equals to yarn configuration value.
	int expectedReplication = flinkFileReplication > 0
		? flinkFileReplication : replication;
	assertEquals(expectedReplication, fsStatus.getReplication());
}
 
Example 3
Source Project: tez   File: TestATSHistoryWithACLs.java    License: Apache License 2.0 6 votes vote down vote up
private void verifyEntityDomains(ApplicationId applicationId, boolean sameDomain) {
  assertNotNull(timelineAddress);

  String appUrl = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_APPLICATION/"
      + "tez_" + applicationId.toString();
  LOG.info("Getting timeline entity for tez application: " + appUrl);
  TimelineEntity appEntity = getTimelineData(appUrl, TimelineEntity.class);

  TezDAGID tezDAGID = TezDAGID.getInstance(applicationId, 1);
  String dagUrl = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/"
      + tezDAGID.toString();
  LOG.info("Getting timeline entity for tez dag: " + dagUrl);
  TimelineEntity dagEntity = getTimelineData(dagUrl, TimelineEntity.class);

  // App and DAG entities should have different domains
  assertEquals(ATSHistoryACLPolicyManager.DOMAIN_ID_PREFIX + applicationId.toString(),
      appEntity.getDomainId());
  if (!sameDomain) {
    assertEquals(ATSHistoryACLPolicyManager.DOMAIN_ID_PREFIX + applicationId.toString()
        + "_1", dagEntity.getDomainId());
  } else {
    assertEquals(appEntity.getDomainId(), dagEntity.getDomainId());
  }
}
 
Example 4
public static ApplicationIdProto convertApplicationIdToApplicationIdProto(
    ApplicationId applicationId) {
  String application = applicationId != null ? applicationId.toString() : "";
  ApplicationIdProto applicationIdProto =
      ApplicationIdProto.newBuilder().setApplicationId(
          application).build();
  return applicationIdProto;
}
 
Example 5
Source Project: tez   File: ATSHistoryACLPolicyManager.java    License: Apache License 2.0 5 votes vote down vote up
private Map<String, String> createDAGDomain(Configuration tezConf,
    ApplicationId applicationId, String dagName, DAGAccessControls dagAccessControls)
    throws IOException, HistoryACLPolicyException {
  String domainId =
      tezConf.get(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID);
  if (!tezConf.getBoolean(TezConfiguration.TEZ_AM_ACLS_ENABLED,
      TezConfiguration.TEZ_AM_ACLS_ENABLED_DEFAULT)) {
    if (domainId != null) {
      throw new TezUncheckedException("ACLs disabled but domainId for DAG is specified"
          + ", aclsEnabled=true, domainId=" + domainId);
    }
    return null;
  }

  boolean autoCreateDomain = tezConf.getBoolean(TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE,
      TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE_DEFAULT);

  if (domainId != null) {
    // do nothing
    LOG.info("Using specified domainId with Timeline, domainId=" + domainId);
  } else {
    if (!autoCreateDomain) {
      // Error - Cannot fallback to default as that leaves ACLs open
      throw new TezUncheckedException("Timeline DomainId is not specified and auto-create"
          + " Domains is disabled");
    }

    // Create a domain only if dagAccessControls has been specified.
    if (dagAccessControls == null) {
      return null;
    }
    domainId = DOMAIN_ID_PREFIX + applicationId.toString() + "_" + dagName;
    createTimelineDomain(domainId, tezConf, dagAccessControls);
    LOG.info("Created Timeline Domain for DAG-specific History ACLs, domainId=" + domainId);
  }
  return Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID, domainId);
}
 
Example 6
private Map<String, String> createDAGDomain(Configuration tezConf,
    ApplicationId applicationId, String dagName, DAGAccessControls dagAccessControls)
    throws IOException, HistoryACLPolicyException {
  String domainId =
      tezConf.get(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID);
  if (!tezConf.getBoolean(TezConfiguration.TEZ_AM_ACLS_ENABLED,
      TezConfiguration.TEZ_AM_ACLS_ENABLED_DEFAULT)) {
    if (domainId != null) {
      throw new TezUncheckedException("ACLs disabled but domainId for DAG is specified"
          + ", aclsEnabled=true, domainId=" + domainId);
    }
    return null;
  }

  boolean autoCreateDomain = tezConf.getBoolean(TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE,
      TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE_DEFAULT);

  if (domainId != null) {
    // do nothing
    LOG.info("Using specified domainId with Timeline, domainId=" + domainId);
  } else {
    if (!autoCreateDomain) {
      // Error - Cannot fallback to default as that leaves ACLs open
      throw new TezUncheckedException("Timeline DomainId is not specified and auto-create"
          + " Domains is disabled");
    }

    // Create a domain only if dagAccessControls has been specified.
    if (dagAccessControls == null) {
      return null;
    }
    domainId = DOMAIN_ID_PREFIX + applicationId.toString() + "_" + dagName;
    createTimelineDomain(applicationId, domainId, tezConf, dagAccessControls);
    LOG.info("Created Timeline Domain for DAG-specific History ACLs, domainId=" + domainId);
  }
  return Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID, domainId);
}
 
Example 7
private Map<String, String> createSessionDomain(Configuration tezConf,
    ApplicationId applicationId, DAGAccessControls dagAccessControls)
    throws IOException, HistoryACLPolicyException {
  String domainId =
      tezConf.get(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID);
  if (!tezConf.getBoolean(TezConfiguration.TEZ_AM_ACLS_ENABLED,
      TezConfiguration.TEZ_AM_ACLS_ENABLED_DEFAULT)) {
    if (domainId != null) {
      throw new TezUncheckedException("ACLs disabled but DomainId is specified"
          + ", aclsEnabled=true, domainId=" + domainId);
    }
    return null;
  }

  boolean autoCreateDomain = tezConf.getBoolean(TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE,
      TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE_DEFAULT);

  if (domainId != null) {
    // do nothing
    LOG.info("Using specified domainId with Timeline, domainId=" + domainId);
  } else {
    if (!autoCreateDomain) {
      // Error - Cannot fallback to default as that leaves ACLs open
      throw new TezUncheckedException("Timeline DomainId is not specified and auto-create"
          + " Domains is disabled");
    }
    domainId = DOMAIN_ID_PREFIX + applicationId.toString();
    createTimelineDomain(applicationId, domainId, tezConf, dagAccessControls);
    LOG.info("Created Timeline Domain for History ACLs, domainId=" + domainId);
  }
  return Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, domainId);
}
 
Example 8
private synchronized YarnContainerClusterOperations buildClusterOperations(RestTemplate restTemplate,
		YarnClient client, ApplicationId applicationId) {
	String key = applicationId.toString();
	YarnContainerClusterOperations operations = operationsCache.get(key);
	if (operations == null) {
		ApplicationReport report = client.getApplicationReport(applicationId);
		String trackingUrl = report.getOriginalTrackingUrl();
		operations = new YarnContainerClusterTemplate(trackingUrl + "/" + YarnContainerClusterEndpoint.ENDPOINT_ID,
				restTemplate);
		operationsCache.put(key, operations);
	}
	return operations;
}
 
Example 9
Source Project: tez   File: ShuffleHandler.java    License: Apache License 2.0 5 votes vote down vote up
private String getDagLocation(String jobId, String dagId, String user) {
  final JobID jobID = JobID.forName(jobId);
  final ApplicationId appID =
      ApplicationId.newInstance(Long.parseLong(jobID.getJtIdentifier()),
          jobID.getId());
  final String dagStr =
      USERCACHE + Path.SEPARATOR + user + Path.SEPARATOR
          + APPCACHE + Path.SEPARATOR
          + appID.toString() + Path.SEPARATOR + Constants.DAG_PREFIX + dagId
          + Path.SEPARATOR;
  return dagStr;
}
 
Example 10
private HistoryFileReader getHistoryFileReader(ApplicationId appId)
    throws IOException {
  Path applicationHistoryFile = new Path(rootDirPath, appId.toString());
  if (!fs.exists(applicationHistoryFile)) {
    throw new IOException("History file for application " + appId
        + " is not found");
  }
  // The history file is still under writing
  if (outstandingWriters.containsKey(appId)) {
    throw new IOException("History file for application " + appId
        + " is under writing");
  }
  return new HistoryFileReader(applicationHistoryFile);
}
 
Example 11
Source Project: zeppelin   File: HadoopUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static void cleanupStagingDirInternal(ClusterClient clusterClient) {
  try {
    ApplicationId appId = (ApplicationId) clusterClient.getClusterId();
    FileSystem fs = FileSystem.get(new Configuration());
    Path stagingDirPath = new Path(fs.getHomeDirectory(), ".flink/" + appId.toString());
    if (fs.delete(stagingDirPath, true)) {
      LOGGER.info("Deleted staging directory " + stagingDirPath);
    }
  } catch (IOException e){
      LOGGER.warn("Failed to cleanup staging dir", e);
  }
}
 
Example 12
Source Project: reef   File: DFSEvaluatorPreserver.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @return the folder for Evaluator changelog.
 */
private static String getEvaluatorChangeLogFolderLocation() {
  final ApplicationId appId = YarnUtilities.getApplicationId();
  if (appId != null) {
    return appId.toString();
  }

  final String jobIdentifier = EvaluatorManager.getJobIdentifier();
  if (jobIdentifier != null) {
    return jobIdentifier;
  }

  throw new RuntimeException("Could not retrieve a suitable DFS folder for preserving Evaluator changelog.");
}
 
Example 13
Source Project: liteflow   File: SparkOnYarnContainer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void run() throws Exception {

    ExecutorJob executorJob = this.getExecutorJob();

    String config = executorJob.getConfig();
    JSONObject configObj = null;
    if(StringUtils.isNotBlank(config)){
        configObj = JSONObject.parseObject(config);
    }

    Tuple<String, String> configTuple = this.generateConfigFile(JSONUtils.toJSONStringWithoutCircleDetect(configObj));
    String configName = configTuple.getA();
    String configPath = configTuple.getB();

    configObj.put(Constants.CONFIG_FILE_NAME, configName);
    configObj.put(Constants.JOB_CONFIG_PATH, configPath);
    /**
     * 初始化spark conf
     */
    this.sparkConf = initSparkConf(configObj);

    /**
     * 生成用户参数
     */
    ClientArguments clientArgs = getArgs(configObj);
    /**
     * 提交到yarn
     */
    Client client = new Client(clientArgs, this.sparkConf);
    ApplicationId applicationId = client.submitApplication();
    String appId = applicationId.toString();
    logger.info("{} get yarn applicationId:{}", executorJob.getId(), appId);
    ExecutorJobService executorJobService = ExecutorServiceUtils.getExecutorJobService();
    /**
     * 这只运行状态
     */
    this.setStatus(ContainerStatus.RUNNING);
    executorJob.setApplicationId(appId);
    executorJobService.bindApplicationIdAndRun(executorJob.getId(), appId);
}
 
Example 14
Source Project: hadoop   File: ConverterUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static String toString(ApplicationId appId) {
  return appId.toString();
}
 
Example 15
Source Project: hadoop   File: TestNMWebServices.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testContainerLogs() throws IOException {
  WebResource r = resource();
  final ContainerId containerId = BuilderUtils.newContainerId(0, 0, 0, 0);
  final String containerIdStr = BuilderUtils.newContainerId(0, 0, 0, 0)
      .toString();
  final ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId();
  final ApplicationId appId = appAttemptId.getApplicationId();
  final String appIdStr = appId.toString();
  final String filename = "logfile1";
  final String logMessage = "log message\n";
  nmContext.getApplications().put(appId, new ApplicationImpl(null, "user",
      appId, null, nmContext));
  
  MockContainer container = new MockContainer(appAttemptId,
      new AsyncDispatcher(), new Configuration(), "user", appId, 1);
  container.setState(ContainerState.RUNNING);
  nmContext.getContainers().put(containerId, container);
  
  // write out log file
  Path path = dirsHandler.getLogPathForWrite(
      ContainerLaunch.getRelativeContainerLogDir(
          appIdStr, containerIdStr) + "/" + filename, false);
  
  File logFile = new File(path.toUri().getPath());
  logFile.deleteOnExit();
  assertTrue("Failed to create log dir", logFile.getParentFile().mkdirs());
  PrintWriter pw = new PrintWriter(logFile);
  pw.print(logMessage);
  pw.close();

  // ask for it
  ClientResponse response = r.path("ws").path("v1").path("node")
      .path("containerlogs").path(containerIdStr).path(filename)
      .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
  String responseText = response.getEntity(String.class);
  assertEquals(logMessage, responseText);
  
  // ask for file that doesn't exist
  response = r.path("ws").path("v1").path("node")
      .path("containerlogs").path(containerIdStr).path("uhhh")
      .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
  Assert.assertEquals(Status.NOT_FOUND.getStatusCode(), response.getStatus());
  responseText = response.getEntity(String.class);
  assertTrue(responseText.contains("Cannot find this log on the local disk."));
  
  // After container is completed, it is removed from nmContext
  nmContext.getContainers().remove(containerId);
  Assert.assertNull(nmContext.getContainers().get(containerId));
  response =
      r.path("ws").path("v1").path("node").path("containerlogs")
          .path(containerIdStr).path(filename).accept(MediaType.TEXT_PLAIN)
          .get(ClientResponse.class);
  responseText = response.getEntity(String.class);
  assertEquals(logMessage, responseText);
}
 
Example 16
public String submitApplication(String applicationName) {
	ApplicationId appId = null;
	appId = yarnClient.submitApplication(
			new ApplicationDescriptor(resolveApplicationdir(springYarnProperties, applicationName)));
	return appId != null ? appId.toString() : null;
}
 
Example 17
Source Project: big-c   File: ConverterUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static String toString(ApplicationId appId) {
  return appId.toString();
}
 
Example 18
Source Project: tez   File: TestDAGAppMaster.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testBadProgress() throws Exception {
  TezConfiguration conf = new TezConfiguration();
  conf.setBoolean(TezConfiguration.TEZ_AM_CREDENTIALS_MERGE, true);
  conf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
  conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, TEST_DIR.toString());
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);

  // create some sample AM credentials
  Credentials amCreds = new Credentials();
  JobTokenSecretManager jtsm = new JobTokenSecretManager();
  JobTokenIdentifier identifier = new JobTokenIdentifier(
      new Text(appId.toString()));
  Token<JobTokenIdentifier> sessionToken =
      new Token<JobTokenIdentifier>(identifier, jtsm);
  sessionToken.setService(identifier.getJobId());
  TokenCache.setSessionToken(sessionToken, amCreds);
  TestTokenSecretManager ttsm = new TestTokenSecretManager();
  Text tokenAlias1 = new Text("alias1");
  Token<TestTokenIdentifier> amToken1 = new Token<TestTokenIdentifier>(
      new TestTokenIdentifier(new Text("amtoken1")), ttsm);
  amCreds.addToken(tokenAlias1, amToken1);

  FileSystem fs = FileSystem.getLocal(conf);
  FSDataOutputStream sessionJarsPBOutStream =
      TezCommonUtils.createFileForAM(fs, new Path(TEST_DIR.toString(),
          TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
  DAGProtos.PlanLocalResourcesProto.getDefaultInstance()
      .writeDelimitedTo(sessionJarsPBOutStream);
  sessionJarsPBOutStream.close();
  DAGAppMaster am = spy(new DAGAppMaster(attemptId,
      ContainerId.newContainerId(attemptId, 1),
      "127.0.0.1", 0, 0, new MonotonicClock(), 1, true,
      TEST_DIR.toString(), new String[] {TEST_DIR.toString()},
      new String[] {TEST_DIR.toString()},
      new TezApiVersionInfo().getVersion(), amCreds,
      "someuser", null));
  when(am.getState()).thenReturn(DAGAppMasterState.RUNNING);
  am.init(conf);
  am.start();
  Credentials dagCreds = new Credentials();
  Token<TestTokenIdentifier> dagToken1 = new Token<TestTokenIdentifier>(
      new TestTokenIdentifier(new Text("dagtoken1")), ttsm);
  dagCreds.addToken(tokenAlias1, dagToken1);
  Text tokenAlias3 = new Text("alias3");
  Token<TestTokenIdentifier> dagToken2 = new Token<TestTokenIdentifier>(
      new TestTokenIdentifier(new Text("dagtoken2")), ttsm);
  dagCreds.addToken(tokenAlias3, dagToken2);
  TezDAGID dagId = TezDAGID.getInstance(appId, 1);
  DAGPlan dagPlan = DAGPlan.newBuilder()
      .setName("somedag")
      .setCredentialsBinary(
          DagTypeConverters.convertCredentialsToProto(dagCreds))
      .build();
  DAGImpl dag = spy(am.createDAG(dagPlan, dagId));
  am.setCurrentDAG(dag);
  when(dag.getState()).thenReturn(DAGState.RUNNING);
  Map<TezVertexID, Vertex> map = new HashMap<TezVertexID, Vertex>();
  TezVertexID mockVertexID = mock(TezVertexID.class);
  Vertex mockVertex = mock(Vertex.class);
  when(mockVertex.getProgress()).thenReturn(Float.NaN);
  map.put(mockVertexID, mockVertex);
  when(dag.getVertices()).thenReturn(map);
  when(dag.getTotalVertices()).thenReturn(1);
  Assert.assertEquals("Progress was NaN and should be reported as 0",
      0, am.getProgress(), 0);
  when(mockVertex.getProgress()).thenReturn(-10f);
  Assert.assertEquals("Progress was negative and should be reported as 0",
      0, am.getProgress(), 0);
  when(mockVertex.getProgress()).thenReturn(1.0000567f);
  Assert.assertEquals(
      "Progress was greater than 1 by a small float precision "
          + "1.0000567 and should be reported as 1",
      1.0f, am.getProgress(), 0.0f);
  when(mockVertex.getProgress()).thenReturn(10f);
  Assert.assertEquals(
      "Progress was greater than 1 and should be reported as 1",
      1.0f, am.getProgress(), 0.0f);
}
 
Example 19
/**
 * Function to run a log handler with directories failing the getFileStatus
 * call. The function accepts the log handler, setup the mocks to fail with
 * specific exceptions and ensures the deletion service has the correct calls.
 * 
 * @param logHandler the logHandler implementation to test
 * 
 * @param appId the application id that we wish when sending events to the log
 * handler
 * 
 * @param user the user name to use
 * 
 * @param mockDelService a mock of the DeletionService which we will verify
 * the delete calls against
 * 
 * @param dirsHandler a spy or mock on the LocalDirsHandler service used to
 * when creating the logHandler. It needs to be a spy so that we can intercept
 * the getAllLogDirs() call.
 * 
 * @param conf the configuration used
 * 
 * @param spylfs a spy on the AbstractFileSystem object used when creating lfs
 * 
 * @param lfs the FileContext object to be used to mock the getFileStatus()
 * calls
 * 
 * @param localLogDirs list of the log dirs to run the test against, must have
 * at least 7 entries
 */
public static void runMockedFailedDirs(LogHandler logHandler,
    ApplicationId appId, String user, DeletionService mockDelService,
    LocalDirsHandlerService dirsHandler, Configuration conf,
    AbstractFileSystem spylfs, FileContext lfs, File[] localLogDirs)
    throws Exception {
  Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
  if (localLogDirs.length < 7) {
    throw new IllegalArgumentException(
      "Argument localLogDirs must be at least of length 7");
  }
  Path[] localAppLogDirPaths = new Path[localLogDirs.length];
  for (int i = 0; i < localAppLogDirPaths.length; i++) {
    localAppLogDirPaths[i] =
        new Path(localLogDirs[i].getAbsolutePath(), appId.toString());
  }
  final List<String> localLogDirPaths =
      new ArrayList<String>(localLogDirs.length);
  for (int i = 0; i < localLogDirs.length; i++) {
    localLogDirPaths.add(localLogDirs[i].getAbsolutePath());
  }

  // setup mocks
  FsPermission defaultPermission =
      FsPermission.getDirDefault().applyUMask(lfs.getUMask());
  final FileStatus fs =
      new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0,
        defaultPermission, "", "",
        new Path(localLogDirs[0].getAbsolutePath()));
  doReturn(fs).when(spylfs).getFileStatus(isA(Path.class));
  doReturn(localLogDirPaths).when(dirsHandler).getLogDirsForCleanup();

  logHandler.handle(new LogHandlerAppStartedEvent(appId, user, null,
    ContainerLogsRetentionPolicy.ALL_CONTAINERS, appAcls));

  // test case where some dirs have the log dir to delete
  // mock some dirs throwing various exceptions
  // verify deletion happens only on the others
  Mockito.doThrow(new FileNotFoundException()).when(spylfs)
    .getFileStatus(eq(localAppLogDirPaths[0]));
  doReturn(fs).when(spylfs).getFileStatus(eq(localAppLogDirPaths[1]));
  Mockito.doThrow(new AccessControlException()).when(spylfs)
    .getFileStatus(eq(localAppLogDirPaths[2]));
  doReturn(fs).when(spylfs).getFileStatus(eq(localAppLogDirPaths[3]));
  Mockito.doThrow(new IOException()).when(spylfs)
    .getFileStatus(eq(localAppLogDirPaths[4]));
  Mockito.doThrow(new UnsupportedFileSystemException("test")).when(spylfs)
    .getFileStatus(eq(localAppLogDirPaths[5]));
  doReturn(fs).when(spylfs).getFileStatus(eq(localAppLogDirPaths[6]));

  logHandler.handle(new LogHandlerAppFinishedEvent(appId));

  testDeletionServiceCall(mockDelService, user, 5000, localAppLogDirPaths[1],
    localAppLogDirPaths[3], localAppLogDirPaths[6]);

  return;
}
 
Example 20
Source Project: hadoop   File: LogAggregationUtils.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Gets the remote app log dir.
 * @param remoteRootLogDir
 * @param appId
 * @param user
 * @param suffix
 * @return the remote application specific log dir.
 */
public static Path getRemoteAppLogDir(Path remoteRootLogDir,
    ApplicationId appId, String user, String suffix) {
  return new Path(getRemoteLogSuffixedDir(remoteRootLogDir, user, suffix),
      appId.toString());
}