Java Code Examples for org.apache.hadoop.mapreduce.v2.hs.HistoryContext

The following examples show how to use org.apache.hadoop.mapreduce.v2.hs.HistoryContext. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestHsWebServicesAttempts.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 2, 1);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
 
Example 2
Source Project: hadoop   Source File: TestHsWebServices.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 1, 1);
  JobHistory jobHistoryService = new JobHistory();
  HistoryContext historyContext = (HistoryContext) jobHistoryService;
  webApp = new HsWebApp(historyContext);

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
 
Example 3
Source Project: hadoop   Source File: TestHsWebServicesJobs.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 2, 1, false);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
 
Example 4
Source Project: hadoop   Source File: TestHsWebServicesJobsQuery.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(3, 2, 1);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
 
Example 5
Source Project: hadoop   Source File: TestHsWebServicesTasks.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 2, 1);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
 
Example 6
Source Project: big-c   Source File: TestHsWebServicesAttempts.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 2, 1);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
 
Example 7
Source Project: big-c   Source File: TestHsWebServices.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 1, 1);
  JobHistory jobHistoryService = new JobHistory();
  HistoryContext historyContext = (HistoryContext) jobHistoryService;
  webApp = new HsWebApp(historyContext);

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
 
Example 8
Source Project: big-c   Source File: TestHsWebServicesJobs.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 2, 1, false);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
 
Example 9
Source Project: big-c   Source File: TestHsWebServicesJobsQuery.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(3, 2, 1);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
 
Example 10
Source Project: big-c   Source File: TestHsWebServicesTasks.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 2, 1);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
 
Example 11
Source Project: XLearning   Source File: HistoryClientService.java    License: Apache License 2.0 5 votes vote down vote up
public HistoryClientService(HistoryContext history,
                            JHSDelegationTokenSecretManager jhsDTSecretManager) {
  super("HistoryClientService");
  this.history = history;
  this.protocolHandler = new HSClientProtocolHandler();
  this.jhsDTSecretManager = jhsDTSecretManager;
}
 
Example 12
Source Project: XLearning   Source File: JobHistoryServer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  Configuration config = new XLearningConfiguration(conf);

  config.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);

  // This is required for WebApps to use https if enabled.
  XLearningWebAppUtil.initialize(getConfig());

  try {
    doSecureLogin(conf);
  } catch (IOException ie) {
    throw new YarnRuntimeException("History Server Failed to login", ie);
  }

  jobHistoryService = new JobHistory();
  historyContext = (HistoryContext) jobHistoryService;
  stateStore = createStateStore(conf);
  this.jhsDTSecretManager = createJHSSecretManager(conf, stateStore);
  clientService = createHistoryClientService();
  aggLogDelService = new AggregatedLogDeletionService();

  addService(stateStore);
  addService(new HistoryServerSecretManagerService());
  addService(clientService);
  addService(aggLogDelService);
  super.serviceInit(config);
}
 
Example 13
Source Project: XLearning   Source File: HsWebApp.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setup() {
  bind(HsWebServices.class);
  bind(JAXBContextResolver.class);
  bind(GenericExceptionHandler.class);
  bind(AppContext.class).toInstance(history);
  bind(HistoryContext.class).toInstance(history);
  route("/", HsController.class);
  route(pajoin("/job", APP_ID), HsController.class, "job");
  route(pajoin("/logs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER,
      CONTAINER_LOG_TYPE), HsController.class, "logs");
}
 
Example 14
Source Project: hadoop   Source File: HsWebApp.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setup() {
  bind(HsWebServices.class);
  bind(JAXBContextResolver.class);
  bind(GenericExceptionHandler.class);
  bind(AppContext.class).toInstance(history);
  bind(HistoryContext.class).toInstance(history);
  route("/", HsController.class);
  route("/app", HsController.class);
  route(pajoin("/job", JOB_ID), HsController.class, "job");
  route(pajoin("/conf", JOB_ID), HsController.class, "conf");
  route(pajoin("/jobcounters", JOB_ID), HsController.class, "jobCounters");
  route(pajoin("/singlejobcounter",JOB_ID, COUNTER_GROUP, COUNTER_NAME),
      HsController.class, "singleJobCounter");
  route(pajoin("/tasks", JOB_ID, TASK_TYPE), HsController.class, "tasks");
  route(pajoin("/attempts", JOB_ID, TASK_TYPE, ATTEMPT_STATE),
      HsController.class, "attempts");
  route(pajoin("/task", TASK_ID), HsController.class, "task");
  route(pajoin("/taskcounters", TASK_ID), HsController.class, "taskCounters");
  route(pajoin("/singletaskcounter",TASK_ID, COUNTER_GROUP, COUNTER_NAME),
      HsController.class, "singleTaskCounter");
  route("/about", HsController.class, "about");
  route(pajoin("/logs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER,
      CONTAINER_LOG_TYPE), HsController.class, "logs");
  route(pajoin("/nmlogs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER,
      CONTAINER_LOG_TYPE), HsController.class, "nmlogs");
}
 
Example 15
Source Project: hadoop   Source File: TestHsWebServicesAcls.java    License: Apache License 2.0 5 votes vote down vote up
private static HistoryContext buildHistoryContext(final Configuration conf)
    throws IOException {
  HistoryContext ctx = new MockHistoryContext(1, 1, 1);
  Map<JobId, Job> jobs = ctx.getAllJobs();
  JobId jobId = jobs.keySet().iterator().next();
  Job mockJob = new MockJobForAcls(jobs.get(jobId), conf);
  jobs.put(jobId, mockJob);
  return ctx;
}
 
Example 16
Source Project: hadoop   Source File: TestHsWebServicesJobs.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testJobCountersForKilledJob() throws Exception {
  WebResource r = resource();
  appContext = new MockHistoryContext(0, 1, 1, 1, true);
  injector = Guice.createInjector(new ServletModule() {
    @Override
    protected void configureServlets() {

      webApp = mock(HsWebApp.class);
      when(webApp.name()).thenReturn("hsmockwebapp");

      bind(JAXBContextResolver.class);
      bind(HsWebServices.class);
      bind(GenericExceptionHandler.class);
      bind(WebApp.class).toInstance(webApp);
      bind(AppContext.class).toInstance(appContext);
      bind(HistoryContext.class).toInstance(appContext);
      bind(Configuration.class).toInstance(conf);

      serve("/*").with(GuiceContainer.class);
    }
  });
  
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("history")
        .path("mapreduce").path("jobs").path(jobId).path("counters/")
        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject info = json.getJSONObject("jobCounters");
    WebServicesTestUtils.checkStringMatch("id", MRApps.toString(id),
        info.getString("id"));
    assertTrue("Job shouldn't contain any counters", info.length() == 1);
  }
}
 
Example 17
Source Project: big-c   Source File: HsWebApp.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setup() {
  bind(HsWebServices.class);
  bind(JAXBContextResolver.class);
  bind(GenericExceptionHandler.class);
  bind(AppContext.class).toInstance(history);
  bind(HistoryContext.class).toInstance(history);
  route("/", HsController.class);
  route("/app", HsController.class);
  route(pajoin("/job", JOB_ID), HsController.class, "job");
  route(pajoin("/conf", JOB_ID), HsController.class, "conf");
  route(pajoin("/jobcounters", JOB_ID), HsController.class, "jobCounters");
  route(pajoin("/singlejobcounter",JOB_ID, COUNTER_GROUP, COUNTER_NAME),
      HsController.class, "singleJobCounter");
  route(pajoin("/tasks", JOB_ID, TASK_TYPE), HsController.class, "tasks");
  route(pajoin("/attempts", JOB_ID, TASK_TYPE, ATTEMPT_STATE),
      HsController.class, "attempts");
  route(pajoin("/task", TASK_ID), HsController.class, "task");
  route(pajoin("/taskcounters", TASK_ID), HsController.class, "taskCounters");
  route(pajoin("/singletaskcounter",TASK_ID, COUNTER_GROUP, COUNTER_NAME),
      HsController.class, "singleTaskCounter");
  route("/about", HsController.class, "about");
  route(pajoin("/logs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER,
      CONTAINER_LOG_TYPE), HsController.class, "logs");
  route(pajoin("/nmlogs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER,
      CONTAINER_LOG_TYPE), HsController.class, "nmlogs");
}
 
Example 18
Source Project: big-c   Source File: TestHsWebServicesAcls.java    License: Apache License 2.0 5 votes vote down vote up
private static HistoryContext buildHistoryContext(final Configuration conf)
    throws IOException {
  HistoryContext ctx = new MockHistoryContext(1, 1, 1);
  Map<JobId, Job> jobs = ctx.getAllJobs();
  JobId jobId = jobs.keySet().iterator().next();
  Job mockJob = new MockJobForAcls(jobs.get(jobId), conf);
  jobs.put(jobId, mockJob);
  return ctx;
}
 
Example 19
Source Project: big-c   Source File: TestHsWebServicesJobs.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testJobCountersForKilledJob() throws Exception {
  WebResource r = resource();
  appContext = new MockHistoryContext(0, 1, 1, 1, true);
  injector = Guice.createInjector(new ServletModule() {
    @Override
    protected void configureServlets() {

      webApp = mock(HsWebApp.class);
      when(webApp.name()).thenReturn("hsmockwebapp");

      bind(JAXBContextResolver.class);
      bind(HsWebServices.class);
      bind(GenericExceptionHandler.class);
      bind(WebApp.class).toInstance(webApp);
      bind(AppContext.class).toInstance(appContext);
      bind(HistoryContext.class).toInstance(appContext);
      bind(Configuration.class).toInstance(conf);

      serve("/*").with(GuiceContainer.class);
    }
  });
  
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("history")
        .path("mapreduce").path("jobs").path(jobId).path("counters/")
        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject info = json.getJSONObject("jobCounters");
    WebServicesTestUtils.checkStringMatch("id", MRApps.toString(id),
        info.getString("id"));
    assertTrue("Job shouldn't contain any counters", info.length() == 1);
  }
}
 
Example 20
Source Project: XLearning   Source File: HsWebApp.java    License: Apache License 2.0 4 votes vote down vote up
public HsWebApp(HistoryContext history) {
  this.history = history;
}
 
Example 21
Source Project: hadoop   Source File: HsWebServices.java    License: Apache License 2.0 4 votes vote down vote up
@Inject
public HsWebServices(final HistoryContext ctx, final Configuration conf,
    final WebApp webapp) {
  this.ctx = ctx;
  this.webapp = webapp;
}
 
Example 22
Source Project: hadoop   Source File: HsWebApp.java    License: Apache License 2.0 4 votes vote down vote up
public HsWebApp(HistoryContext history) {
  this.history = history;
}
 
Example 23
Source Project: hadoop   Source File: TestHsWebServicesJobConf.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void configureServlets() {

  Path confPath = new Path(testConfDir.toString(),
      MRJobConfig.JOB_CONF_FILE);
  Configuration config = new Configuration();

  FileSystem localFs;
  try {
    localFs = FileSystem.getLocal(config);
    confPath = localFs.makeQualified(confPath);

    OutputStream out = localFs.create(confPath);
    try {
      conf.writeXml(out);
    } finally {
      out.close();
    }
    if (!localFs.exists(confPath)) {
      fail("error creating config file: " + confPath);
    }

  } catch (IOException e) {
    fail("error creating config file: " + e.getMessage());
  }

  appContext = new MockHistoryContext(0, 2, 1, confPath);

  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
 
Example 24
Source Project: big-c   Source File: HsWebServices.java    License: Apache License 2.0 4 votes vote down vote up
@Inject
public HsWebServices(final HistoryContext ctx, final Configuration conf,
    final WebApp webapp) {
  this.ctx = ctx;
  this.webapp = webapp;
}
 
Example 25
Source Project: big-c   Source File: HsWebApp.java    License: Apache License 2.0 4 votes vote down vote up
public HsWebApp(HistoryContext history) {
  this.history = history;
}
 
Example 26
Source Project: big-c   Source File: TestHsWebServicesJobConf.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void configureServlets() {

  Path confPath = new Path(testConfDir.toString(),
      MRJobConfig.JOB_CONF_FILE);
  Configuration config = new Configuration();

  FileSystem localFs;
  try {
    localFs = FileSystem.getLocal(config);
    confPath = localFs.makeQualified(confPath);

    OutputStream out = localFs.create(confPath);
    try {
      conf.writeXml(out);
    } finally {
      out.close();
    }
    if (!localFs.exists(confPath)) {
      fail("error creating config file: " + confPath);
    }

  } catch (IOException e) {
    fail("error creating config file: " + e.getMessage());
  }

  appContext = new MockHistoryContext(0, 2, 1, confPath);

  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}