org.apache.reef.util.EnvironmentUtils Java Examples

The following examples show how to use org.apache.reef.util.EnvironmentUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TaskMessagingTest.java    From reef with Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskMsg() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfig = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "DriverTaskMsg")
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, TaskMessagingDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, TaskMessagingDriver.TaskRunningHandler.class)
      .set(DriverConfiguration.ON_TASK_MESSAGE, TaskMessagingDriver.TaskMessageHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfig, this.testEnvironment.getTestTimeout());

  Assert.assertEquals(LauncherStatus.COMPLETED, status);
}
 
Example #2
Source File: FailDriverDelayedMsgTest.java    From reef with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailDriverTestMessage() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfig = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "FailDriverDelayedMsg")
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, FailDriverDelayedMsg.AllocatedEvaluatorHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, FailDriverDelayedMsg.ActiveContextHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, FailDriverDelayedMsg.RunningTaskHandler.class)
      .set(DriverConfiguration.ON_TASK_MESSAGE, FailDriverDelayedMsg.TaskMessageHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STARTED, FailDriverDelayedMsg.StartHandler.class)
      .build();

  final LauncherStatus status = TestDriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfig, this.testEnvironment.getTestTimeout());

  Assert.assertEquals(LauncherStatus.COMPLETED, status);
}
 
Example #3
Source File: EvaluatorFailureTest.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that only an EvaluatorFailure is generated by exceptions thrown on by an Alarm handler.
 */
@Test
public void testEvaluatorFailureByAlarmHandler() throws InjectionException {
  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES,
          EnvironmentUtils.getClassLocation(EvaluatorFailureDuringAlarmDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_EvaluatorFailureTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, EvaluatorFailureDuringAlarmDriver.ActiveContextHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED,
          EvaluatorFailureDuringAlarmDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_FAILED, EvaluatorFailureDuringAlarmDriver.EvaluatorFailureHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_FAILED, EvaluatorFailureDuringAlarmDriver.ContextFailureHandler.class)
      .set(DriverConfiguration.ON_TASK_FAILED, EvaluatorFailureDuringAlarmDriver.TaskFailureHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STOP, EvaluatorFailureDuringAlarmDriver.StopHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("EvaluatorFailureTest.testEvaluatorFailureByAlarmHandler() state = " + status,
      status.isSuccess());
}
 
Example #4
Source File: DriverTest.java    From reef with Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleDriver() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(DriverTestStartHandler.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_DriverTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, DriverTestStartHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 
Example #5
Source File: RogueThreadTest.java    From reef with Apache License 2.0 6 votes vote down vote up
@Test
public void testRogueThread() throws InjectionException {
  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();
  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "Test_RogueThreadTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, RogueThreadDriver.EvaluatorAllocationHandler.class)
      .set(DriverConfiguration.ON_TASK_FAILED, ExpectedTaskFailureHandler.class)
      .build();
  final LauncherStatus state = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + state, state.isSuccess());


}
 
Example #6
Source File: StatePassingTest.java    From reef with Apache License 2.0 6 votes vote down vote up
@Test
public void testStatePassing() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "StatePassingTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, StatePassingDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, StatePassingDriver.ContextActiveHandler.class)
      .set(DriverConfiguration.ON_TASK_COMPLETED, StatePassingDriver.TaskCompletedHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 
Example #7
Source File: CloseEvaluatorTest.java    From reef with Apache License 2.0 6 votes vote down vote up
@Test
public void testCloseEvaluator() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(CloseEvaluatorDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_CloseEvaluatorTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, CloseEvaluatorDriver.StartHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 
Example #8
Source File: EvaluatorCloseTest.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that an evaluator's state is changed to closing and closed in order during closing.
 */
@Test
public void testEvaluatorClosingState() throws InjectionException {
  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES,
          EnvironmentUtils.getClassLocation(EvaluatorCloseDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_EvaluatorCloseTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, EvaluatorCloseDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, EvaluatorCloseDriver.TaskRunningHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STOP, EvaluatorCloseDriver.StopHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("The result state of evaluator closing state test is " + status,
      status.isSuccess());
}
 
Example #9
Source File: RackAwareEvaluatorTest.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
* Tests whether the runtime passes the rack information to the driver.
* The success scenario is if it receives the default rack, fails otherwise.
*/
@Test
public void testRackAwareEvaluatorRunningOnDefaultRack() {
  //Given
  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_RackAwareEvaluator")
      .set(DriverConfiguration.GLOBAL_LIBRARIES,
          EnvironmentUtils.getClassLocation(RackAwareEvaluatorTestDriver.class))
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, RackAwareEvaluatorTestDriver.EvaluatorAllocatedHandler.class)
      .build();

  // When
  final LauncherStatus status = this.testEnvironment.run(driverConfiguration);
  // Then
  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 
Example #10
Source File: JobLauncher.java    From nemo with Apache License 2.0 6 votes vote down vote up
/**
 * Get driver configuration.
 * @param jobConf job Configuration to get job id and driver memory.
 * @return driver configuration.
 * @throws InjectionException exception while injection.
 */
private static Configuration getDriverConf(final Configuration jobConf) throws InjectionException {
  final Injector injector = TANG.newInjector(jobConf);
  final String jobId = injector.getNamedInstance(JobConf.JobId.class);
  final int driverMemory = injector.getNamedInstance(JobConf.DriverMemMb.class);
  return DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(NemoDriver.class))
      .set(DriverConfiguration.ON_DRIVER_STARTED, NemoDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, NemoDriver.AllocatedEvaluatorHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, NemoDriver.ActiveContextHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_FAILED, NemoDriver.FailedEvaluatorHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_FAILED, NemoDriver.FailedContextHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STOP, NemoDriver.DriverStopHandler.class)
      .set(DriverConfiguration.DRIVER_IDENTIFIER, jobId)
      .set(DriverConfiguration.DRIVER_MEMORY, driverMemory)
      .build();
}
 
Example #11
Source File: TaskResubmitTest.java    From reef with Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskResubmission() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_TaskResubmitTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, TaskResubmitDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_TASK_FAILED, TaskResubmitDriver.TaskFailedHandler.class)
      .build();

  final LauncherStatus state = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + state, state.isSuccess());
}
 
Example #12
Source File: EvaluatorSizeTest.java    From reef with Apache License 2.0 6 votes vote down vote up
private LauncherStatus runEvaluatorSizeTest(final int megaBytes) throws BindException, InjectionException {
  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration testConfiguration = EvaluatorSizeTestConfiguration.CONF
      .set(EvaluatorSizeTestConfiguration.MEMORY_SIZE, 777)
      .build();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_EvaluatorSizeTest-" + megaBytes)
      .set(DriverConfiguration.ON_DRIVER_STARTED, EvaluatorSizeTestDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED,
          EvaluatorSizeTestDriver.EvaluatorAllocatedHandler.class).build();

  final Configuration mergedDriverConfiguration = Tang.Factory.getTang()
      .newConfigurationBuilder(driverConfiguration, testConfiguration).build();

  final LauncherStatus state = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(mergedDriverConfiguration, this.testEnvironment.getTestTimeout());
  return state;
}
 
Example #13
Source File: REEFEnvironment.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * Launch REEF component (Driver or Evaluator).
 * It is usually called from the static .run() method.
 * Check the status of the run via .getLastStatus() method.
 */
@Override
@SuppressWarnings("checkstyle:illegalcatch") // Catch throwable to feed it to error handler
public void run() {

  LOG.log(Level.FINE, "REEF started with user name [{0}]", System.getProperty("user.name"));
  LOG.log(Level.FINE, "REEF started. Assertions are {0} in this process.",
          EnvironmentUtils.areAssertionsEnabled() ? "ENABLED" : "DISABLED");

  try {

    LOG.log(Level.FINEST, "Clock: start");
    this.clock.run();
    LOG.log(Level.FINEST, "Clock: exit normally: {0}", this.getLastStatus());

  } catch (final Throwable ex) {
    LOG.log(Level.SEVERE, "Clock: Error in main event loop", ex);
    this.errorHandler.onNext(ex);
  }
}
 
Example #14
Source File: BGDClient.java    From reef with Apache License 2.0 6 votes vote down vote up
private Configuration getDataLoadConfiguration(final String jobName) {
  final EvaluatorRequest computeRequest = EvaluatorRequest.newBuilder()
      .setNumber(1)
      .setMemory(memory)
      .build();
  final Configuration dataLoadConfiguration = new DataLoadingRequestBuilder()
      .setInputFormatClass(TextInputFormat.class)
      .setInputPath(input)
      .setNumberOfDesiredSplits(numSplits)
      .addComputeRequest(computeRequest)
      .renewFailedEvaluators(false)
      .setDriverConfigurationModule(DriverConfiguration.CONF
          .setMultiple(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getAllClasspathJars())
          .set(DriverConfiguration.DRIVER_MEMORY, Integer.toString(memory))
          .set(DriverConfiguration.ON_CONTEXT_ACTIVE, BGDDriver.ContextActiveHandler.class)
          .set(DriverConfiguration.ON_TASK_RUNNING, BGDDriver.TaskRunningHandler.class)
          .set(DriverConfiguration.ON_TASK_FAILED, BGDDriver.TaskFailedHandler.class)
          .set(DriverConfiguration.ON_TASK_COMPLETED, BGDDriver.TaskCompletedHandler.class)
          .set(DriverConfiguration.DRIVER_IDENTIFIER, jobName))
      .build();
  return dataLoadConfiguration;
}
 
Example #15
Source File: HelloREEFHttp.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * @return the configuration of the HelloREEF driver.
 */
public static Configuration getDriverConfiguration() {
  return DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(HttpShellJobDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "HelloREEF")
      .set(DriverConfiguration.ON_DRIVER_STARTED, HttpShellJobDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, HttpShellJobDriver.AllocatedEvaluatorHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_FAILED, HttpShellJobDriver.FailedEvaluatorHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, HttpShellJobDriver.ActiveContextHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_CLOSED, HttpShellJobDriver.ClosedContextHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_FAILED, HttpShellJobDriver.FailedContextHandler.class)
      .set(DriverConfiguration.ON_TASK_COMPLETED, HttpShellJobDriver.CompletedTaskHandler.class)
      .set(DriverConfiguration.ON_CLIENT_MESSAGE, HttpShellJobDriver.ClientMessageHandler.class)
      .set(DriverConfiguration.ON_CLIENT_CLOSED, HttpShellJobDriver.HttpClientCloseHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STOP, HttpShellJobDriver.StopHandler.class)
      .build();
}
 
Example #16
Source File: VortexJobConf.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * Instantiate a {@link VortexJobConf} object, where a Configuration is built by Tang internally.
 *
 * {@link IllegalArgumentException} will be thrown if required parameters are not set
 * (See {@link #setJobName(String)} and {@link #setVortexMasterConf(Configuration)}).
 *
 * Also, {@link org.apache.reef.tang.exceptions.BindException} can be thrown while merging the configurations.
 *
 * @return An instance of VortexJobConf object.
 */
@Override
public VortexJobConf build() {
  BuilderUtils.notNull(jobName);
  BuilderUtils.notNull(vortexMasterConf);

  final Configuration vortexDriverConf = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(VortexDriver.class))
      .set(DriverConfiguration.ON_DRIVER_STARTED, VortexDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, VortexDriver.AllocatedEvaluatorHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, VortexDriver.RunningTaskHandler.class)
      .set(DriverConfiguration.ON_TASK_MESSAGE, VortexDriver.TaskMessageHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_FAILED, VortexDriver.FailedEvaluatorHandler.class)
      .set(DriverConfiguration.DRIVER_IDENTIFIER, jobName)
      .build();

  final Configuration jobConf;
  if (userConf.isPresent()) {
    jobConf = Configurations.merge(vortexDriverConf, vortexMasterConf, userConf.get());
  } else {
    jobConf = Configurations.merge(vortexDriverConf, vortexMasterConf);
  }
  return new VortexJobConf(jobConf);
}
 
Example #17
Source File: HelloREEF.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * Start Hello REEF job with local runtime.
 * @param args command line parameters.
 * @throws InjectionException configuration error.
 */
public static void main(final String[] args) throws InjectionException, IOException {

  final ClientProtocol.DriverClientConfiguration.Builder builder =
      ClientProtocol.DriverClientConfiguration.newBuilder();
  builder.setJobid("HelloREEF");
  builder.setEnableHttpDriver(false);
  builder.setOperatingSystem(ClientProtocol.DriverClientConfiguration.OS.LINUX);
  builder.setAzbatchRuntime(ClientProtocol.AzureBatchRuntimeParameters.newBuilder()
      .build());
  builder.addGlobalLibraries(EnvironmentUtils.getClassLocation(HelloDriver.class));

  JavaClientLauncher.submit(builder.build(), DRIVER_CONFIG);
  LOG.log(Level.INFO, "REEF job completed");
  ThreadLogger.logThreads(LOG, Level.FINE, "Threads running at the end of HelloREEF:");
}
 
Example #18
Source File: JobClient.java    From reef with Apache License 2.0 6 votes vote down vote up
public static ConfigurationModule getDriverConfiguration() {
  return DriverConfiguration.CONF
      .setMultiple(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getAllClasspathJars())
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, JobDriver.AllocatedEvaluatorHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_FAILED, JobDriver.FailedEvaluatorHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, JobDriver.ActiveContextHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_CLOSED, JobDriver.ClosedContextHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_FAILED, JobDriver.FailedContextHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_MESSAGE, JobDriver.ContextMessageHandler.class)
      .set(DriverConfiguration.ON_TASK_MESSAGE, JobDriver.TaskMessageHandler.class)
      .set(DriverConfiguration.ON_TASK_FAILED, JobDriver.FailedTaskHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, JobDriver.RunningTaskHandler.class)
      .set(DriverConfiguration.ON_TASK_COMPLETED, JobDriver.CompletedTaskHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STARTED, JobDriver.StartHandler.class)
      .set(DriverConfiguration.ON_TASK_SUSPENDED, JobDriver.SuspendedTaskHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_COMPLETED, JobDriver.CompletedEvaluatorHandler.class);
}
 
Example #19
Source File: TestGroupCommServiceInjection.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * Run the GroupCommServiceInjection test.
 */
@Test
public void testGroupCommServiceInjection() {
  final Configuration driverConf = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES,
          EnvironmentUtils.getClassLocation(GroupCommServiceInjectionDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER,
          "TEST_GroupCommServiceInjection")
      .set(DriverConfiguration.ON_DRIVER_STARTED,
          GroupCommServiceInjectionDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED,
          GroupCommServiceInjectionDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE,
          GroupCommServiceInjectionDriver.ContextActiveHandler.class)
      .build();

  final Configuration groupCommConf = GroupCommService.getConfiguration();
  final LauncherStatus state = this.testEnvironment.run(Configurations.merge(driverConf, groupCommConf));
  Assert.assertTrue("Job state after execution: " + state, state.isSuccess());
}
 
Example #20
Source File: JobLauncher.java    From incubator-nemo with Apache License 2.0 6 votes vote down vote up
/**
 * Get driver configuration.
 *
 * @param jobConf job Configuration to get job id and driver memory.
 * @return driver configuration.
 * @throws InjectionException exception while injection.
 */
private static Configuration getDriverConf(final Configuration jobConf) throws InjectionException {
  final Injector injector = TANG.newInjector(jobConf);
  final String jobId = injector.getNamedInstance(JobConf.JobId.class);
  final int driverMemory = injector.getNamedInstance(JobConf.DriverMemMb.class);
  return DriverConfiguration.CONF
    .setMultiple(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getAllClasspathJars())
    .set(DriverConfiguration.ON_DRIVER_STARTED, NemoDriver.StartHandler.class)
    .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, NemoDriver.AllocatedEvaluatorHandler.class)
    .set(DriverConfiguration.ON_CONTEXT_ACTIVE, NemoDriver.ActiveContextHandler.class)
    .set(DriverConfiguration.ON_EVALUATOR_FAILED, NemoDriver.FailedEvaluatorHandler.class)
    .set(DriverConfiguration.ON_CONTEXT_FAILED, NemoDriver.FailedContextHandler.class)
    .set(DriverConfiguration.ON_DRIVER_STOP, NemoDriver.DriverStopHandler.class)
    .set(DriverConfiguration.DRIVER_IDENTIFIER, jobId)
    .set(DriverConfiguration.DRIVER_MEMORY, driverMemory)
    .build();
}
 
Example #21
Source File: FailureREEF.java    From reef with Apache License 2.0 6 votes vote down vote up
public static LauncherStatus runFailureReef(
    final Configuration runtimeConfig, final int timeout, final int numEvaluatorsToSubmit,
    final int numEvaluatorsToFail) throws InjectionException {

  final Configuration driverConf = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(FailureDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "FailureREEF")
      .set(DriverConfiguration.ON_DRIVER_STARTED, FailureDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, FailureDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_FAILED, FailureDriver.EvaluatorFailedHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STOP, FailureDriver.StopHandler.class)
      .build();

  final Configuration namedParamsConf = Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(NumEvaluatorsToSubmit.class, Integer.toString(numEvaluatorsToSubmit))
      .bindNamedParameter(NumEvaluatorsToFail.class, Integer.toString(numEvaluatorsToFail))
      .build();

  final LauncherStatus state = DriverLauncher.getLauncher(runtimeConfig)
      .run(Configurations.merge(driverConf, namedParamsConf), timeout);

  LOG.log(Level.INFO, "REEF job completed: {0}", state);
  return state;
}
 
Example #22
Source File: FailClient.java    From reef with Apache License 2.0 6 votes vote down vote up
private static Configuration buildDriverConfig(final Class<?> failMsgClass) {

    final Configuration driverConfig = DriverConfiguration.CONF
        .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(FailDriver.class))
        .set(DriverConfiguration.DRIVER_IDENTIFIER, "Fail_" + failMsgClass.getSimpleName())
        .set(DriverConfiguration.ON_DRIVER_STARTED, FailDriver.StartHandler.class)
        .set(DriverConfiguration.ON_DRIVER_STOP, FailDriver.StopHandler.class)
        .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, FailDriver.AllocatedEvaluatorHandler.class)
        .set(DriverConfiguration.ON_EVALUATOR_COMPLETED, FailDriver.CompletedEvaluatorHandler.class)
        .set(DriverConfiguration.ON_EVALUATOR_FAILED, FailDriver.FailedEvaluatorHandler.class)
        .set(DriverConfiguration.ON_CONTEXT_ACTIVE, FailDriver.ActiveContextHandler.class)
        .set(DriverConfiguration.ON_CONTEXT_MESSAGE, FailDriver.ContextMessageHandler.class)
        .set(DriverConfiguration.ON_CONTEXT_CLOSED, FailDriver.ClosedContextHandler.class)
        .set(DriverConfiguration.ON_CONTEXT_FAILED, FailDriver.FailedContextHandler.class)
        .set(DriverConfiguration.ON_TASK_RUNNING, FailDriver.RunningTaskHandler.class)
        .set(DriverConfiguration.ON_TASK_SUSPENDED, FailDriver.SuspendedTaskHandler.class)
        .set(DriverConfiguration.ON_TASK_MESSAGE, FailDriver.TaskMessageHandler.class)
        .set(DriverConfiguration.ON_TASK_FAILED, FailDriver.FailedTaskHandler.class)
        .set(DriverConfiguration.ON_TASK_COMPLETED, FailDriver.CompletedTaskHandler.class)
        .build();

    return TANG.newConfigurationBuilder(driverConfig)
        .bindNamedParameter(FailDriver.FailMsgClassName.class, failMsgClass.getName())
        .build();
  }
 
Example #23
Source File: Client.java    From reef with Apache License 2.0 6 votes vote down vote up
public static LauncherStatus run(
    final Class<? extends Task> failTaskClass,
    final Configuration runtimeConfig,
    final int timeOut) throws BindException, InjectionException {

  final Configuration driverConfig = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(Driver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, failTaskClass.getSimpleName())
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, Driver.AllocatedEvaluatorHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, Driver.RunningTaskHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, Driver.ActiveContextHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STARTED, Driver.StartHandler.class)
      .build();

  final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder();
  cb.addConfiguration(driverConfig);
  cb.bindNamedParameter(Driver.FailTaskName.class, failTaskClass.getSimpleName());

  return TestDriverLauncher.getLauncher(runtimeConfig).run(cb.build(), timeOut);
}
 
Example #24
Source File: RuntimeNameTest.java    From reef with Apache License 2.0 6 votes vote down vote up
@Test
public void testRuntimeName() throws BindException, InjectionException {
  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();
  final Configuration testConfiguration = RuntimeNameTestConfiguration.CONF
      .set(RuntimeNameTestConfiguration.RUNTIME_NAME, this.testEnvironment.getRuntimeName())
      .build();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(OnDriverStartedAllocateOne.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_DriverTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, RuntimeNameDriver.EvaluatorAllocatedHandler.class)
      .build();

  final Configuration mergedDriverConfiguration = Tang.Factory.getTang()
      .newConfigurationBuilder(driverConfiguration, testConfiguration).build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(mergedDriverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 
Example #25
Source File: WatcherTest.java    From reef with Apache License 2.0 6 votes vote down vote up
private Configuration getDriverConfiguration() {
  final Configuration driverConf = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "WatcherTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, WatcherTestDriver.DriverStartedHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, WatcherTestDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_FAILED, WatcherTestDriver.EvaluatorFailedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, WatcherTestDriver.ContextActivatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_FAILED, WatcherTestDriver.ContextFailedHandler.class)
      .set(DriverConfiguration.ON_TASK_FAILED, WatcherTestDriver.TaskFailedHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, WatcherTestDriver.TaskRunningHandler.class)
      .set(DriverConfiguration.ON_TASK_SUSPENDED, WatcherTestDriver.TaskSuspendedHandler.class)
      .build();

  final Configuration runtimeStopConf = Tang.Factory.getTang().newConfigurationBuilder()
      .bindSetEntry(Clock.RuntimeStopHandler.class, WatcherTestDriver.RuntimeStopHandler.class)
      .build();

  return Configurations.merge(driverConf, runtimeStopConf, WatcherConfiguration.CONF
      .set(WatcherConfiguration.EVENT_STREAMS, LogEventStream.class)
      .set(WatcherConfiguration.EVENT_STREAMS, TestEventStream.class)
      .build());
}
 
Example #26
Source File: EvaluatorReuseTest.java    From reef with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluatorReuse() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(EvaluatorReuseTestDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_EvaluatorReuseTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_TASK_COMPLETED, EvaluatorReuseTestDriver.TaskCompletedHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, EvaluatorReuseTestDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, EvaluatorReuseTestDriver.ContextActiveHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("EvaluatorReuse state = " + status, status.isSuccess());
}
 
Example #27
Source File: DriverFailOnFailTest.java    From reef with Apache License 2.0 6 votes vote down vote up
@Test
public void testDriverFailOnFail() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfig = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "Fail2")
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, DriverFailOnFail.AllocatedEvaluatorHandler.class)
      .set(DriverConfiguration.ON_TASK_FAILED, DriverFailOnFail.FailedTaskHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STARTED, DriverFailOnFail.StartHandler.class)
      .build();

  TestUtils.assertLauncherFailure(
      TestDriverLauncher.getLauncher(runtimeConfiguration).run(
          driverConfig, this.testEnvironment.getTestTimeout()),
      SimulatedDriverFailure.class);
}
 
Example #28
Source File: SubContextTest.java    From reef with Apache License 2.0 6 votes vote down vote up
@Test
public void testSubContexts() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_SubContextTest_testSubContexts")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, SubContextDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, SubContextDriver.ContextActiveHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_CLOSED, SubContextDriver.ContextClosedHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 
Example #29
Source File: SubContextTest.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * Same as testSubContexts(), but using default ClosedContext handler.
 */
@Test
public void testChainClose() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_SubContextTest_testChainClose")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, SubContextDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, SubContextDriver.ContextActiveHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 
Example #30
Source File: TestMultipleCommGroups.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * Run the MultipleCommGroups test.
 */
@Test
public void testMultipleCommGroups() {
  final Configuration driverConf = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(MultipleCommGroupsDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_MultipleCommGroups")
      .set(DriverConfiguration.ON_DRIVER_STARTED, MultipleCommGroupsDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, MultipleCommGroupsDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, MultipleCommGroupsDriver.ContextActiveHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, MultipleCommGroupsDriver.TaskRunningHandler.class)
      .set(DriverConfiguration.ON_TASK_COMPLETED, MultipleCommGroupsDriver.TaskCompletedHandler.class)
      .build();
  final Configuration groupCommConf = GroupCommService.getConfiguration();
  final LauncherStatus state = this.testEnvironment.run(Configurations.merge(driverConf, groupCommConf));
  Assert.assertTrue("Job state after execution: " + state, state.isSuccess());
}