org.apache.reef.client.DriverLauncher Java Examples

/**
 * Launch application using the application DAG.
 * @param dag the application DAG.
 */
// When modifying the signature of this method, see CompilerTestUtil#compileDAG and make corresponding changes
public static void launchDAG(final DAG dag) {
  try {
    if (jobAndDriverConf == null || deployModeConf == null) {
      throw new RuntimeException("Configuration for launching driver is not ready");
    }
    final String serializedDAG = Base64.getEncoder().encodeToString(SerializationUtils.serialize(dag));
    final Configuration dagConf = TANG.newConfigurationBuilder()
        .bindNamedParameter(JobConf.SerializedDAG.class, serializedDAG)
        .build();
    // Launch and wait indefinitely for the job to finish
    final LauncherStatus launcherStatus =  DriverLauncher.getLauncher(deployModeConf)
        .run(Configurations.merge(jobAndDriverConf, dagConf));
    final Optional<Throwable> possibleError = launcherStatus.getError();
    if (possibleError.isPresent()) {
      throw new RuntimeException(possibleError.get());
    } else {
      LOG.info("Job successfully completed");
    }
  } catch (final InjectionException e) {
    throw new RuntimeException(e);
  }
}
 

public static void main(final String[] args)
    throws InjectionException, BindException, IOException {

  final Tang tang = Tang.Factory.getTang();
  final JavaConfigurationBuilder cb = tang.newConfigurationBuilder();
  new CommandLine(cb)
      .registerShortNameOfClass(Local.class)
      .registerShortNameOfClass(TimeOut.class)
      .registerShortNameOfClass(OutputDir.class)
      .processCommandLine(args);

  final Injector injector = tang.newInjector(cb.build());
  final boolean isLocal = injector.getNamedInstance(Local.class);
  final String outputDir = injector.getNamedInstance(OutputDir.class);
  final int jobTimeout = injector.getNamedInstance(TimeOut.class) * 60 * 1000;

  final Configuration driverConf = getDriverConf();
  final Configuration outputServiceConf = getOutputServiceConf(isLocal, outputDir);
  final Configuration submittedConfiguration = Tang.Factory.getTang()
      .newConfigurationBuilder(driverConf, outputServiceConf)
      .build();
  final LauncherStatus state = DriverLauncher.getLauncher(getRuntimeConf(isLocal))
      .run(submittedConfiguration, jobTimeout);

  LOG.log(Level.INFO, "REEF job completed: {0}", state);
}
 

/**
 * Start Hello REEF job with Unmanaged Driver running locally in the same process.
 * @param args command line parameters. Not used.
 * @throws InjectionException configuration error.
 */
public static void main(final String[] args) throws InjectionException {

  LOG.log(Level.FINE, "Launching Unmanaged AM: {0}", JAR_PATH);

  try (DriverLauncher client = DriverLauncher.getLauncher(RUNTIME_CONFIG)) {

    final String appId = client.submit(DRIVER_CONFIG, 10000);
    LOG.log(Level.INFO, "Job submitted: {0}", appId);

    final Configuration yarnAmConfig = UnmanagedAmYarnDriverConfiguration.CONF
        .set(UnmanagedAmYarnDriverConfiguration.JOB_IDENTIFIER, appId)
        .set(UnmanagedAmYarnDriverConfiguration.JOB_SUBMISSION_DIRECTORY, DRIVER_ROOT_PATH)
        .build();

    try (REEFEnvironment reef = REEFEnvironment.fromConfiguration(yarnAmConfig, DRIVER_CONFIG)) {
      reef.run();
      final ReefServiceProtos.JobStatusProto status = reef.getLastStatus();
      LOG.log(Level.INFO, "REEF job {0} completed: state {1}", new Object[] {appId, status.getState()});
    }
  }

  ThreadLogger.logThreads(LOG, Level.FINEST, "Threads running after DriverLauncher.close():");
  System.exit(0); // TODO[REEF-1715]: Should be able to exit cleanly at the end of main()
}
 

private LauncherStatus runEvaluatorSizeTest(final int megaBytes) throws BindException, InjectionException {
  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration testConfiguration = EvaluatorSizeTestConfiguration.CONF
      .set(EvaluatorSizeTestConfiguration.MEMORY_SIZE, 777)
      .build();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_EvaluatorSizeTest-" + megaBytes)
      .set(DriverConfiguration.ON_DRIVER_STARTED, EvaluatorSizeTestDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED,
          EvaluatorSizeTestDriver.EvaluatorAllocatedHandler.class).build();

  final Configuration mergedDriverConfiguration = Tang.Factory.getTang()
      .newConfigurationBuilder(driverConfiguration, testConfiguration).build();

  final LauncherStatus state = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(mergedDriverConfiguration, this.testEnvironment.getTestTimeout());
  return state;
}
 

@Test
public void testTaskResubmission() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_TaskResubmitTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, TaskResubmitDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_TASK_FAILED, TaskResubmitDriver.TaskFailedHandler.class)
      .build();

  final LauncherStatus state = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + state, state.isSuccess());
}
 

@Test
public void testTaskMsg() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfig = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "DriverTaskMsg")
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, TaskMessagingDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, TaskMessagingDriver.TaskRunningHandler.class)
      .set(DriverConfiguration.ON_TASK_MESSAGE, TaskMessagingDriver.TaskMessageHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfig, this.testEnvironment.getTestTimeout());

  Assert.assertEquals(LauncherStatus.COMPLETED, status);
}
 

/**
 * Tests that an evaluator's state is changed to closing and closed in order during closing.
 */
@Test
public void testEvaluatorClosingState() throws InjectionException {
  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES,
          EnvironmentUtils.getClassLocation(EvaluatorCloseDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_EvaluatorCloseTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, EvaluatorCloseDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, EvaluatorCloseDriver.TaskRunningHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STOP, EvaluatorCloseDriver.StopHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("The result state of evaluator closing state test is " + status,
      status.isSuccess());
}
 

@Test
public void testCloseEvaluator() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(CloseEvaluatorDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_CloseEvaluatorTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, CloseEvaluatorDriver.StartHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 

@Test
public void testStatePassing() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "StatePassingTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, StatePassingDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, StatePassingDriver.ContextActiveHandler.class)
      .set(DriverConfiguration.ON_TASK_COMPLETED, StatePassingDriver.TaskCompletedHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 

@Test
public void testRogueThread() throws InjectionException {
  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();
  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "Test_RogueThreadTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, RogueThreadDriver.EvaluatorAllocationHandler.class)
      .set(DriverConfiguration.ON_TASK_FAILED, ExpectedTaskFailureHandler.class)
      .build();
  final LauncherStatus state = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + state, state.isSuccess());


}
 

@Test
public void testSimpleDriver() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(DriverTestStartHandler.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_DriverTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, DriverTestStartHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 

/**
 * Tests that only an EvaluatorFailure is generated by exceptions thrown on by an Alarm handler.
 */
@Test
public void testEvaluatorFailureByAlarmHandler() throws InjectionException {
  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES,
          EnvironmentUtils.getClassLocation(EvaluatorFailureDuringAlarmDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_EvaluatorFailureTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, EvaluatorFailureDuringAlarmDriver.ActiveContextHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED,
          EvaluatorFailureDuringAlarmDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_FAILED, EvaluatorFailureDuringAlarmDriver.EvaluatorFailureHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_FAILED, EvaluatorFailureDuringAlarmDriver.ContextFailureHandler.class)
      .set(DriverConfiguration.ON_TASK_FAILED, EvaluatorFailureDuringAlarmDriver.TaskFailureHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STOP, EvaluatorFailureDuringAlarmDriver.StopHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("EvaluatorFailureTest.testEvaluatorFailureByAlarmHandler() state = " + status,
      status.isSuccess());
}
 

@Test
public void testHelloREEFMultiRuntime() throws InjectionException {
  if(this.testEnvironment instanceof YarnTestEnvironment){
    // multi runtime can be tested on yarn only
    final Configuration driverConf = DriverConfiguration.CONF
            .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
            .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_HelloREEFMultiRunitme")
            .set(DriverConfiguration.ON_DRIVER_STARTED, HelloMultiRuntimeDriver.StartHandler.class)
            .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, HelloMultiRuntimeDriver.EvaluatorAllocatedHandler.class)
            .build();

    // create the multi runtime environment
    Configuration multiruntimeConfig = new MultiRuntimeConfigurationBuilder()
            .addRuntime(RuntimeIdentifier.RUNTIME_NAME)
            .addRuntime(org.apache.reef.runtime.local.driver.RuntimeIdentifier.RUNTIME_NAME)
            .setDefaultRuntime(RuntimeIdentifier.RUNTIME_NAME)
            .setMaxEvaluatorsNumberForLocalRuntime(1)
            .setSubmissionRuntime(RuntimeIdentifier.RUNTIME_NAME)
            .build();

    final LauncherStatus state = DriverLauncher.getLauncher(multiruntimeConfig).run(driverConf, this
            .testEnvironment.getTestTimeout());
    Assert.assertTrue("Job state after execution: " + state, state.isSuccess());
  }
}
 

/**
 * Same as testSubContexts(), but using default ClosedContext handler.
 */
@Test
public void testChainClose() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_SubContextTest_testChainClose")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, SubContextDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, SubContextDriver.ContextActiveHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 

@Test
public void testSubContexts() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_SubContextTest_testSubContexts")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, SubContextDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, SubContextDriver.ContextActiveHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_CLOSED, SubContextDriver.ContextClosedHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 

@Test
public void testEvaluatorReuse() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(EvaluatorReuseTestDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_EvaluatorReuseTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_TASK_COMPLETED, EvaluatorReuseTestDriver.TaskCompletedHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, EvaluatorReuseTestDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, EvaluatorReuseTestDriver.ContextActiveHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("EvaluatorReuse state = " + status, status.isSuccess());
}
 

@Test
public void testRuntimeName() throws BindException, InjectionException {
  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();
  final Configuration testConfiguration = RuntimeNameTestConfiguration.CONF
      .set(RuntimeNameTestConfiguration.RUNTIME_NAME, this.testEnvironment.getRuntimeName())
      .build();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(OnDriverStartedAllocateOne.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_DriverTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, RuntimeNameDriver.EvaluatorAllocatedHandler.class)
      .build();

  final Configuration mergedDriverConfiguration = Tang.Factory.getTang()
      .newConfigurationBuilder(driverConfiguration, testConfiguration).build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(mergedDriverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 

public static LauncherStatus runFailureReef(
    final Configuration runtimeConfig, final int timeout, final int numEvaluatorsToSubmit,
    final int numEvaluatorsToFail) throws InjectionException {

  final Configuration driverConf = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(FailureDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "FailureREEF")
      .set(DriverConfiguration.ON_DRIVER_STARTED, FailureDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, FailureDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_FAILED, FailureDriver.EvaluatorFailedHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STOP, FailureDriver.StopHandler.class)
      .build();

  final Configuration namedParamsConf = Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(NumEvaluatorsToSubmit.class, Integer.toString(numEvaluatorsToSubmit))
      .bindNamedParameter(NumEvaluatorsToFail.class, Integer.toString(numEvaluatorsToFail))
      .build();

  final LauncherStatus state = DriverLauncher.getLauncher(runtimeConfig)
      .run(Configurations.merge(driverConf, namedParamsConf), timeout);

  LOG.log(Level.INFO, "REEF job completed: {0}", state);
  return state;
}
 

/**
 * Start Hello REEF job.
 *
 * @param args command line parameters.
 * @throws BindException      configuration error.
 * @throws InjectionException configuration error.
 */
public static void main(final String[] args) throws BindException, InjectionException {
  final Configuration runtimeConf = getHybridYarnSubmissionRuntimeConfiguration();
  final Configuration driverConf = getDriverConfiguration();

  final LauncherStatus status = DriverLauncher
      .getLauncher(runtimeConf)
      .run(driverConf, JOB_TIMEOUT);
  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}
 

private static LauncherStatus launch(final Configuration runtimeConf, final Configuration vortexConf) {
  try {
    return DriverLauncher.getLauncher(runtimeConf).run(vortexConf);
  } catch (InjectionException e) {
    throw new RuntimeException(e);
  }
}
 

/**
 * MASTER_IP  is set to "localhost:5050".
 * You may change it to suit your cluster environment.
 *
 * @param args command line parameters.
 * @throws BindException      configuration error.
 * @throws InjectionException configuration error.
 */
public static void main(final String[] args) throws BindException, InjectionException {
  final Configuration runtimeConf = getRuntimeConfiguration();
  final Configuration driverConf = getDriverConfiguration();

  final LauncherStatus status = DriverLauncher
      .getLauncher(runtimeConf)
      .run(driverConf);
  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}
 

/**
 * Start HelloJVMOptions REEF job.
 *
 * @param args command line parameters.
 * @throws BindException      configuration error.
 * @throws InjectionException configuration error.
 */
public static void main(final String[] args) throws BindException, InjectionException {
  final Configuration runtimeConf = getRuntimeConfiguration();
  final Configuration driverConf = getDriverConfiguration();

  final LauncherStatus status = DriverLauncher
      .getLauncher(runtimeConf)
      .run(driverConf, JOB_TIMEOUT);
  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}
 

/**
 * Start Hello REEF job.
 *
 * @param args command line parameters.
 * @throws BindException      configuration error.
 * @throws InjectionException configuration error.
 */
public static void main(final String[] args) throws BindException, InjectionException {
  final int tcpBeginPort = args.length > 0 ? Integer.parseInt(args[0]) : DEFAULT_TCP_BEGIN_PORT;
  final int tcpRangeCount = args.length > 1 ? Integer.parseInt(args[1]) : DEFAULT_TCP_RANGE_COUNT;
  final int tcpTryCount = args.length > 2 ? Integer.parseInt(args[2]) : DEFAULT_TCP_RANGE_TRY_COUNT;

  final Configuration runtimeConf = getRuntimeConfiguration(tcpBeginPort, tcpRangeCount, tcpTryCount);
  final Configuration driverConf = getDriverConfiguration();

  final LauncherStatus status = DriverLauncher
      .getLauncher(runtimeConf)
      .run(driverConf, JOB_TIMEOUT);
  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}
 

/**
 * Start Hello REEF job.
 *
 * @param args command line parameters.
 * @throws BindException      configuration error.
 * @throws InjectionException configuration error.
 */
public static void main(final String[] args) throws BindException, InjectionException {

  final Tang tang = Tang.Factory.getTang();

  final JavaConfigurationBuilder cb = tang.newConfigurationBuilder();

  try{
    new CommandLine(cb)
        .registerShortNameOfClass(NodeListFilePath.class)
        .registerShortNameOfClass(SshPortNum.class)
        .processCommandLine(args);
  } catch(final IOException ex) {
    LOG.log(Level.SEVERE, "Missing parameter 'nodelist' or wrong parameter input.");
    throw new RuntimeException("Missing parameter 'nodelist' or wrong parameter input: ", ex);
  }

  final Injector injector = tang.newInjector(cb.build());

  final String nodeListFilePath = injector.getNamedInstance(NodeListFilePath.class);
  final int sshPortNum = injector.getNamedInstance(SshPortNum.class);

  final Configuration runtimeConf = getRuntimeConfiguration(nodeListFilePath, sshPortNum);
  final Configuration driverConf = getDriverConfiguration();

  final LauncherStatus status = DriverLauncher
      .getLauncher(runtimeConf)
      .run(driverConf, JOB_TIMEOUT);
  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}
 

/**
 * Start HelloREEFYarnRestart job.
 *
 * @param args command line parameters.
 * @throws org.apache.reef.tang.exceptions.BindException      configuration error.
 * @throws org.apache.reef.tang.exceptions.InjectionException configuration error.
 */
public static void main(final String[] args) throws InjectionException {
  final Configuration runtimeConf = getRuntimeConfiguration();
  final Configuration driverConf = getDriverConfiguration();

  final LauncherStatus status = DriverLauncher
      .getLauncher(runtimeConf)
      .run(driverConf, JOB_TIMEOUT);
  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}
 

public static LauncherStatus runHelloCLR(final Configuration runtimeConf, final int timeOut, final File clrFolder)
    throws BindException, InjectionException {

  final ConfigurationModule driverConf =
      addAll(DriverConfiguration.CONF, DriverConfiguration.GLOBAL_FILES, clrFolder)
          .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(HelloDriver.class))
          .set(DriverConfiguration.DRIVER_IDENTIFIER, "HelloCLR")
          .set(DriverConfiguration.ON_DRIVER_STARTED, HelloDriver.StartHandler.class)
          .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, HelloDriver.EvaluatorAllocatedHandler.class);

  return DriverLauncher.getLauncher(runtimeConf).run(driverConf.build(), timeOut);
}
 

/** StartTime event: launch another REEF job. */
@Override
public void onNext(final StartTime startTime) {

  LOG.log(Level.INFO, "Driver started: app {0} :: {1}", new Object[] {this.hostApplicationId, startTime});
  LOG.log(Level.FINE, "Launching Unmanaged AM: {0}", JAR_PATH);

  try (DriverLauncher client = DriverLauncher.getLauncher(RUNTIME_CONFIG)) {

    final String innerApplicationId = client.submit(DRIVER_CONFIG, 10000);
    LOG.log(Level.INFO, "REEF-on-REEF job submitted: host app {0} new app {1}",
        new Object[] {this.hostApplicationId, innerApplicationId});

    final Configuration yarnAmConfig = UnmanagedAmYarnDriverConfiguration.CONF
        .set(UnmanagedAmYarnDriverConfiguration.JOB_IDENTIFIER, innerApplicationId)
        .set(UnmanagedAmYarnDriverConfiguration.JOB_SUBMISSION_DIRECTORY, DRIVER_ROOT_PATH)
        .build();

    try (REEFEnvironment reef =
        REEFEnvironment.fromConfiguration(client.getUser(), yarnAmConfig, DRIVER_CONFIG)) {

      reef.run();

      final ReefServiceProtos.JobStatusProto status = reef.getLastStatus();

      LOG.log(Level.INFO, "REEF-on-REEF inner job {0} completed: state {1}",
          new Object[] {innerApplicationId, status.getState()});
    }

    LOG.log(Level.INFO,
        "REEF-on-REEF host job {0} completed: inner app {1} status {2}",
        new Object[] {this.hostApplicationId, innerApplicationId, client.getStatus()});

  } catch (final InjectionException ex) {
    LOG.log(Level.SEVERE, "REEF-on-REEF configuration error", ex);
    throw new RuntimeException("REEF-on-REEF configuration error", ex);
  }

  ThreadLogger.logThreads(LOG, Level.FINEST, "Threads running after DriverLauncher.close():");
}
 

public static LauncherStatus runBGDReef(
    final Configuration runtimeConfiguration) throws InjectionException {

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .setMultiple(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getAllClasspathJars())
      .set(DriverConfiguration.ON_DRIVER_STARTED, BroadcastDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, BroadcastDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, BroadcastDriver.ContextActiveHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_CLOSED, BroadcastDriver.ContextCloseHandler.class)
      .set(DriverConfiguration.ON_TASK_FAILED, BroadcastDriver.FailedTaskHandler.class)
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "BroadcastDriver")
      .build();

  final Configuration groupCommServConfiguration = GroupCommService.getConfiguration();

  final Configuration mergedDriverConfiguration = Tang.Factory.getTang()
      .newConfigurationBuilder(groupCommServConfiguration, driverConfiguration)
      .bindNamedParameter(ModelDimensions.class, Integer.toString(dimensions))
      .bindNamedParameter(NumberOfReceivers.class, Integer.toString(numberOfReceivers))
      .build();

  if (LOG.isLoggable(Level.FINE)) {
    LOG.log(Level.FINE, "Merged driver configuration:\n{0}",
        Configurations.toString(mergedDriverConfiguration));
  }

  return DriverLauncher.getLauncher(runtimeConfiguration).run(mergedDriverConfiguration, JOB_TIMEOUT);
}
 

@Test
public void testDriverFiles() throws IOException, BindException, InjectionException {

  final Set<File> theFiles = getTempFiles(this.nFiles);
  final Configuration finalDriverConfiguration = Configurations.merge(
      getDriverConfiguration(theFiles), getTestDriverConfiguration(theFiles));

  final LauncherStatus status = DriverLauncher
      .getLauncher(this.testEnvironment.getRuntimeConfiguration())
      .run(finalDriverConfiguration, testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 

/**
 * Tests whether parameters get propagated correctly when set via the ConfigurationProvider mechanism.
 *
 * @throws InjectionException
 */
@Test
public void testConfigurationProviders() throws InjectionException {
  final LauncherStatus status = DriverLauncher.getLauncher(getRuntimeConfiguration())
      .run(getDriverConfiguration(), testEnvironment.getTestTimeout());
  Assert.assertTrue("ConfigurationProviderTest completed with status: " + status, status.isSuccess());
}