org.apache.reef.tang.Configurations Java Examples

/**
 * Submit a task to the ActiveContext.
 */
public synchronized void submitTask(final ActiveContext context) {
  final TaskEntity task = taskQueue.poll();
  final Integer taskId = task.getId();
  final String command = task.getCommand();

  final Configuration taskConf = TaskConfiguration.CONF
      .set(TaskConfiguration.TASK, ShellTask.class)
      .set(TaskConfiguration.IDENTIFIER, taskId.toString())
      .build();
  final Configuration commandConf = Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(Command.class, command)
      .build();

  final Configuration merged = Configurations.merge(taskConf, commandConf);
  context.submitTask(merged);
  runningTasks.add(task);
}
 

@Override
public Configuration getDriverConfiguration(final URI jobFolder,
                                            final String clientRemoteId,
                                            final String jobId,
                                            final Configuration applicationConfiguration) {

  final Configuration hdinsightDriverConfiguration = HDInsightDriverConfiguration.CONF
          .set(HDInsightDriverConfiguration.JOB_IDENTIFIER, jobId)
          .set(HDInsightDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId)
          .set(HDInsightDriverConfiguration.JOB_SUBMISSION_DIRECTORY, jobFolder.toString())
          .set(HDInsightDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack)
          .set(HDInsightDriverConfiguration.RUNTIME_NAMES, RuntimeIdentifier.RUNTIME_NAME)
          .build();

  return Configurations.merge(
          applicationConfiguration,
          hdinsightDriverConfiguration);
}
 

@Override
public Configuration getDriverConfiguration(final URI jobFolder,
                                            final String clientRemoteId,
                                            final String jobId,
                                            final Configuration applicationConfiguration) {
  return Configurations.merge(MesosDriverConfiguration.CONF
                  .set(MesosDriverConfiguration.MESOS_MASTER_IP, this.masterIp)
                  .set(MesosDriverConfiguration.JOB_IDENTIFIER, jobId)
                  .set(MesosDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId)
                  .set(MesosDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack)
                  .set(MesosDriverConfiguration.SCHEDULER_DRIVER_CAPACITY, 1)
                  // must be 1 as there is 1 scheduler at the same time
                  .set(MesosDriverConfiguration.RUNTIME_NAMES, RuntimeIdentifier.RUNTIME_NAME)
                  .build(),
          applicationConfiguration);
}
 

/**
 * Instantiate REEF DriverServiceLauncher. This method is called from REEFLauncher.main().
 * @param clockConfigPath Path to the local file that contains serialized configuration
 *                        for the driver client.
 * @return An instance of the configured REEFLauncher object.
 */
private static JavaDriverClientLauncher getLauncher(final String clockConfigPath, final int driverServicePort) {
  try {
    final Configuration clockArgConfig = Configurations.merge(
        LAUNCHER_STATIC_CONFIG,
        DriverClientGrpcConfiguration.CONF
            .set(DriverClientGrpcConfiguration.DRIVER_SERVICE_PORT, driverServicePort)
            .build(),
        TANG.newConfigurationBuilder()
            .bindNamedParameter(ClockConfigurationPath.class, clockConfigPath)
            .build());

    return TANG.newInjector(clockArgConfig).getInstance(JavaDriverClientLauncher.class);
  } catch (final InjectionException ex) {
    throw fatal("Unable to instantiate REEFLauncher.", ex);
  }
}
 

/**
 * Assembles the driver configuration.
 *
 * @param jobFolder                The folder in which the local runtime will execute this job.
 * @param clientRemoteId           the remote identifier of the client. It is used by the Driver to establish a
 *                                 connection back to the client.
 * @param jobId                    The identifier of the job.
 * @param applicationConfiguration The configuration of the application, e.g. a filled out DriverConfiguration
 * @return The Driver configuration to be used to instantiate the Driver.
 */
@Override
public Configuration getDriverConfiguration(final URI jobFolder,
                                            final String clientRemoteId,
                                            final String jobId,
                                            final Configuration applicationConfiguration) {
  AvroMultiRuntimeDefinition runtimeDefinitions = this.definitionGenerator.getMultiRuntimeDefinition(
          jobFolder,
          clientRemoteId,
          jobId);
  ConfigurationModule conf = MultiRuntimeDriverConfiguration.CONF;

  for(AvroRuntimeDefinition runtimeDefinition : runtimeDefinitions.getRuntimes()){
    conf = conf.set(MultiRuntimeDriverConfiguration.RUNTIME_NAMES, runtimeDefinition.getRuntimeName().toString());
  }

  final Configuration mainConfiguration = this.mainRuntimeConfigGenerator.getMainConfiguration();

  return Configurations.merge(mainConfiguration, applicationConfiguration,
                  conf
                  .set(MultiRuntimeDriverConfiguration.JOB_IDENTIFIER, jobId)
                  .set(MultiRuntimeDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId)
                  .set(MultiRuntimeDriverConfiguration.SERIALIZED_RUNTIME_DEFINITION,
                          this.runtimeDefinitionSerializer.toString(runtimeDefinitions))
                  .build());
}
 

@Override
public Configuration getDriverServiceConfiguration(
    final ClientProtocol.DriverClientConfiguration driverConfiguration) {
  Configuration driverServiceConfiguration = DriverServiceConfiguration.CONF
      .set(DriverServiceConfiguration.DRIVER_SERVICE_IMPL, GRPCDriverService.class)
      .set(DriverServiceConfiguration.DRIVER_IDLENESS_SOURCES, GRPCDriverService.class)
      .set(DriverServiceConfiguration.DRIVER_CLIENT_COMMAND, driverConfiguration.getDriverClientLaunchCommand())
      .build();
  return driverConfiguration.getDriverRestartEnable() ?
      Configurations.merge(
          driverServiceConfiguration,
          getDriverRestartConfiguration(driverConfiguration),
          getDriverConfiguration(driverConfiguration),
          getTcpPortRangeConfiguration(driverConfiguration)) :
      Configurations.merge(
          driverServiceConfiguration,
          getDriverConfiguration(driverConfiguration),
          getTcpPortRangeConfiguration(driverConfiguration));
}
 

private Configuration getExecutorConfiguration(final String executorId) {
  final Configuration executorConfiguration = JobConf.EXECUTOR_CONF
    .set(JobConf.EXECUTOR_ID, executorId)
    .set(JobConf.GLUSTER_DISK_DIRECTORY, glusterDirectory)
    .set(JobConf.LOCAL_DISK_DIRECTORY, localDirectory)
    .set(JobConf.JOB_ID, jobId)
    .build();

  final Configuration contextConfiguration = ContextConfiguration.CONF
    .set(ContextConfiguration.IDENTIFIER, executorId) // We set: contextId = executorId
    .set(ContextConfiguration.ON_CONTEXT_STARTED, NemoContext.ContextStartHandler.class)
    .set(ContextConfiguration.ON_CONTEXT_STOP, NemoContext.ContextStopHandler.class)
    .build();

  final Configuration ncsConfiguration = getExecutorNcsConfiguration();
  final Configuration messageConfiguration = getExecutorMessageConfiguration(executorId);
  final Configuration dataPlaneConfiguration = dataPlaneConf.getDataPlaneConfiguration();

  return Configurations.merge(executorConfiguration, contextConfiguration, ncsConfiguration,
    messageConfiguration, dataPlaneConfiguration);
}
 

/**
 * Instantiate a {@link VortexJobConf} object, where a Configuration is built by Tang internally.
 *
 * {@link IllegalArgumentException} will be thrown if required parameters are not set
 * (See {@link #setJobName(String)} and {@link #setVortexMasterConf(Configuration)}).
 *
 * Also, {@link org.apache.reef.tang.exceptions.BindException} can be thrown while merging the configurations.
 *
 * @return An instance of VortexJobConf object.
 */
@Override
public VortexJobConf build() {
  BuilderUtils.notNull(jobName);
  BuilderUtils.notNull(vortexMasterConf);

  final Configuration vortexDriverConf = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(VortexDriver.class))
      .set(DriverConfiguration.ON_DRIVER_STARTED, VortexDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, VortexDriver.AllocatedEvaluatorHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, VortexDriver.RunningTaskHandler.class)
      .set(DriverConfiguration.ON_TASK_MESSAGE, VortexDriver.TaskMessageHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_FAILED, VortexDriver.FailedEvaluatorHandler.class)
      .set(DriverConfiguration.DRIVER_IDENTIFIER, jobName)
      .build();

  final Configuration jobConf;
  if (userConf.isPresent()) {
    jobConf = Configurations.merge(vortexDriverConf, vortexMasterConf, userConf.get());
  } else {
    jobConf = Configurations.merge(vortexDriverConf, vortexMasterConf);
  }
  return new VortexJobConf(jobConf);
}
 

@Override
public void onNext(final AllocatedEvaluator allocatedEvaluator) {
  LOG.log(Level.INFO, "Container allocated");
  final String workerId = allocatedEvaluator.getId() + "_vortex_worker";

  final Configuration workerConfiguration = VortexWorkerConf.CONF
      .set(VortexWorkerConf.NUM_OF_THREADS, evalCores) // NUM_OF_THREADS = evalCores
      .build();

  final Configuration taskConfiguration = TaskConfiguration.CONF
      .set(TaskConfiguration.IDENTIFIER, workerId)
      .set(TaskConfiguration.TASK, VortexWorker.class)
      .set(TaskConfiguration.ON_SEND_MESSAGE, VortexWorker.class)
      .set(TaskConfiguration.ON_MESSAGE, VortexWorker.DriverMessageHandler.class)
      .set(TaskConfiguration.ON_CLOSE, VortexWorker.TaskCloseHandler.class)
      .build();

  allocatedEvaluator.submitTask(Configurations.merge(workerConfiguration, taskConfiguration));
}
 

/**
 * @return the driver-side configuration to be merged into the DriverConfiguration to enable the HTTP server.
 */
public static Configuration getHTTPConfiguration() {
  final Configuration httpHandlerConfiguration = HttpHandlerConfiguration.CONF
      .set(HttpHandlerConfiguration.HTTP_HANDLERS, HttpServerReefEventHandler.class)
      .set(HttpHandlerConfiguration.HTTP_HANDLERS, HttpServerShellCmdHandler.class)
      .build();
  final Configuration driverConfigurationForHttpServer = DriverServiceConfiguration.CONF
      .set(DriverServiceConfiguration.ON_EVALUATOR_ALLOCATED,
          ReefEventStateManager.AllocatedEvaluatorStateHandler.class)
      .set(DriverServiceConfiguration.ON_CONTEXT_ACTIVE, ReefEventStateManager.ActiveContextStateHandler.class)
      .set(DriverServiceConfiguration.ON_TASK_RUNNING, ReefEventStateManager.TaskRunningStateHandler.class)
      .set(DriverServiceConfiguration.ON_DRIVER_STARTED, ReefEventStateManager.StartStateHandler.class)
      .set(DriverServiceConfiguration.ON_DRIVER_STOP, ReefEventStateManager.StopStateHandler.class)
      .build();
  return Configurations.merge(httpHandlerConfiguration, driverConfigurationForHttpServer);
}
 

/**
 * @return the configuration of the HelloREEF driver.
 */
private static Configuration getDriverConfiguration() {
  return
      Configurations.merge(DriverConfiguration.CONF
              .set(DriverConfiguration.GLOBAL_LIBRARIES,
                  HelloREEFYarnRestart.class.getProtectionDomain().getCodeSource().getLocation().getFile())
              .set(DriverConfiguration.DRIVER_IDENTIFIER, "HelloREEF")
              .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, HelloDriver.EvaluatorAllocatedHandler.class)
              .build(),
          YarnDriverRestartConfiguration.CONF
              .build(),
          DriverRestartConfiguration.CONF
              .set(DriverRestartConfiguration.ON_DRIVER_RESTARTED,
                  HelloDriverRestart.DriverRestartHandler.class)
              .build());
}
 

@Test
public void httpServerSpecifiedPortTest() throws Exception {

  final Configuration httpRuntimeConfiguration = HttpRuntimeConfiguration.CONF.build();

  final Configuration httpServerConfiguration = Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(TcpPortRangeBegin.class, "9000")
      .build();

  final Configuration configuration =
      Configurations.merge(httpRuntimeConfiguration, httpServerConfiguration);

  final Injector injector = Tang.Factory.getTang().newInjector(configuration);
  final HttpServer httpServer = injector.getInstance(HttpServer.class);
  Assert.assertNotNull(httpServer);
  httpServer.stop();
}
 

private Configuration getExecutorConfiguration(final String executorId, final int executorCapacity) {
  final Configuration executorConfiguration = JobConf.EXECUTOR_CONF
      .set(JobConf.EXECUTOR_ID, executorId)
      .set(JobConf.EXECUTOR_CAPACITY, executorCapacity)
      .set(JobConf.GLUSTER_DISK_DIRECTORY, glusterDirectory)
      .set(JobConf.LOCAL_DISK_DIRECTORY, localDirectory)
      .set(JobConf.JOB_ID, jobId)
      .build();

  final Configuration contextConfiguration = ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, executorId) // We set: contextId = executorId
      .set(ContextConfiguration.ON_CONTEXT_STARTED, NemoContext.ContextStartHandler.class)
      .set(ContextConfiguration.ON_CONTEXT_STOP, NemoContext.ContextStopHandler.class)
      .build();

  final Configuration ncsConfiguration =  getExecutorNcsConfiguration();
  final Configuration messageConfiguration = getExecutorMessageConfiguration(executorId);

  return Configurations.merge(executorConfiguration, contextConfiguration, ncsConfiguration, messageConfiguration);
}
 

/**
 * @return the driver-side configuration to be merged into the DriverConfiguration to enable the HTTP server.
 */
public static Configuration getHTTPConfiguration() {
  final Configuration httpHandlerConfiguration = HttpHandlerConfiguration.CONF
      .set(HttpHandlerConfiguration.HTTP_HANDLERS, HttpServerReefEventHandler.class)
      .build();

  final Configuration driverConfigurationForHttpServer = DriverServiceConfiguration.CONF
      .set(DriverServiceConfiguration.ON_EVALUATOR_ALLOCATED,
          ReefEventStateManager.AllocatedEvaluatorStateHandler.class)
      .set(DriverServiceConfiguration.ON_CONTEXT_ACTIVE, ReefEventStateManager.ActiveContextStateHandler.class)
      .set(DriverServiceConfiguration.ON_TASK_RUNNING, ReefEventStateManager.TaskRunningStateHandler.class)
      .set(DriverServiceConfiguration.ON_DRIVER_STARTED, ReefEventStateManager.StartStateHandler.class)
      .set(DriverServiceConfiguration.ON_DRIVER_STOP, ReefEventStateManager.StopStateHandler.class)
      .build();

  return Configurations.merge(httpHandlerConfiguration, driverConfigurationForHttpServer);
}
 

private Configuration getDriverConfiguration() {
  final Configuration driverConf = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "WatcherTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, WatcherTestDriver.DriverStartedHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, WatcherTestDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_FAILED, WatcherTestDriver.EvaluatorFailedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, WatcherTestDriver.ContextActivatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_FAILED, WatcherTestDriver.ContextFailedHandler.class)
      .set(DriverConfiguration.ON_TASK_FAILED, WatcherTestDriver.TaskFailedHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, WatcherTestDriver.TaskRunningHandler.class)
      .set(DriverConfiguration.ON_TASK_SUSPENDED, WatcherTestDriver.TaskSuspendedHandler.class)
      .build();

  final Configuration runtimeStopConf = Tang.Factory.getTang().newConfigurationBuilder()
      .bindSetEntry(Clock.RuntimeStopHandler.class, WatcherTestDriver.RuntimeStopHandler.class)
      .build();

  return Configurations.merge(driverConf, runtimeStopConf, WatcherConfiguration.CONF
      .set(WatcherConfiguration.EVENT_STREAMS, LogEventStream.class)
      .set(WatcherConfiguration.EVENT_STREAMS, TestEventStream.class)
      .build());
}
 

/**
 * @return the runtime configuration, based on the parameters passed from C#.
 */
Configuration getRuntimeConfiguration() {
  final Configuration runtimeConfiguration = LocalRuntimeConfiguration.CONF
      .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, Integer.toString(maxNumberOfConcurrentEvaluators))
      .set(LocalRuntimeConfiguration.RUNTIME_ROOT_FOLDER, runtimeRootFolder.getAbsolutePath())
      .build();

  final ArrayList<String> driverLaunchCommandPrefixList = new ArrayList<>();
  driverLaunchCommandPrefixList.add(
      new File(driverFolder,
          new REEFFileNames().getDriverLauncherExeFile().toString()
      ).toString());

  final Configuration userProviderConfiguration = Tang.Factory.getTang().newConfigurationBuilder()
      .bindSetEntry(DriverConfigurationProviders.class, TcpPortConfigurationProvider.class)
      .bindSetEntry(DriverConfigurationProviders.class, LoopbackLocalAddressProvider.class)
      .bindNamedParameter(TcpPortRangeBegin.class, Integer.toString(tcpBeginPort))
      .bindNamedParameter(TcpPortRangeCount.class, Integer.toString(tcpRangeCount))
      .bindNamedParameter(TcpPortRangeTryCount.class, Integer.toString(tcpTryCount))
      .bindNamedParameter(JobSubmissionDirectory.class, runtimeRootFolder.getAbsolutePath())
      .bindList(DriverLaunchCommandPrefix.class, driverLaunchCommandPrefixList)
      .build();

  return Configurations.merge(runtimeConfiguration, userProviderConfiguration);
}
 

@Test
public void httpServerPortRangeTestWithTcpPortProvider() throws Exception {
  final Configuration httpRuntimeConfiguration = HttpRuntimeConfiguration.CONF.build();

  final Configuration httpServerConfiguration = Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(TcpPortRangeCount.class, "8900")
      .bindNamedParameter(TcpPortRangeBegin.class, "1000")
      .bindNamedParameter(TcpPortRangeTryCount.class, "3")
      .build();

  final Configuration configuration =
      Configurations.merge(httpRuntimeConfiguration, httpServerConfiguration);

  final Injector injector1 = Tang.Factory.getTang().newInjector(configuration);
  final HttpServer httpServer1 = injector1.getInstance(HttpServer.class);

  final Injector injector2 = Tang.Factory.getTang().newInjector(configuration);
  final HttpServer httpServer2 = injector2.getInstance(HttpServer.class);

  Assert.assertTrue("port number is out of specified range",
      httpServer2.getPort() >= 1000 && httpServer2.getPort() <= 9900);

  httpServer1.stop();
  httpServer2.stop();
}
 

/**
 * Run the GroupCommServiceInjection test.
 */
@Test
public void testGroupCommServiceInjection() {
  final Configuration driverConf = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES,
          EnvironmentUtils.getClassLocation(GroupCommServiceInjectionDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER,
          "TEST_GroupCommServiceInjection")
      .set(DriverConfiguration.ON_DRIVER_STARTED,
          GroupCommServiceInjectionDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED,
          GroupCommServiceInjectionDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE,
          GroupCommServiceInjectionDriver.ContextActiveHandler.class)
      .build();

  final Configuration groupCommConf = GroupCommService.getConfiguration();
  final LauncherStatus state = this.testEnvironment.run(Configurations.merge(driverConf, groupCommConf));
  Assert.assertTrue("Job state after execution: " + state, state.isSuccess());
}
 

/**
 * Run the MultipleCommGroups test.
 */
@Test
public void testMultipleCommGroups() {
  final Configuration driverConf = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(MultipleCommGroupsDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_MultipleCommGroups")
      .set(DriverConfiguration.ON_DRIVER_STARTED, MultipleCommGroupsDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, MultipleCommGroupsDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, MultipleCommGroupsDriver.ContextActiveHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, MultipleCommGroupsDriver.TaskRunningHandler.class)
      .set(DriverConfiguration.ON_TASK_COMPLETED, MultipleCommGroupsDriver.TaskCompletedHandler.class)
      .build();
  final Configuration groupCommConf = GroupCommService.getConfiguration();
  final LauncherStatus state = this.testEnvironment.run(Configurations.merge(driverConf, groupCommConf));
  Assert.assertTrue("Job state after execution: " + state, state.isSuccess());
}
 

@Before
public void setUp() throws InjectionException, IOException, ServletException {

  this.request = new Request(
      new HttpConnection(new LocalConnector(), new StringEndPoint(), new Server()));

  this.request.setContentType("text/json");

  this.response = new Response(
      new HttpConnection(new LocalConnector(), new StringEndPoint(), new Server()));

  final Configuration httpHandlerConfiguration = HttpHandlerConfiguration.CONF
      .set(HttpHandlerConfiguration.HTTP_HANDLERS, HttpServerReefEventHandler.class)
      .build();

  final Tang tang = Tang.Factory.getTang();

  final Configuration remoteConfiguration = tang.newConfigurationBuilder()
      .bindNamedParameter(RemoteConfiguration.ManagerName.class, "REEF_TEST_REMOTE_MANAGER")
      .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
      .bindNamedParameter(JobIdentifier.class, "my job")
      .build();

  final Configuration finalConfig =
      Configurations.merge(httpHandlerConfiguration, remoteConfiguration);

  final Injector injector = tang.newInjector(finalConfig);

  this.handler = injector.getInstance(JettyHandler.class);
}
 

public static Configuration getYarnConfiguration() {
  final Configuration yarnDriverRestartConfiguration = YarnDriverRestartConfiguration.CONF
      .build();

  final Configuration driverRestartHandlerConfigurations = DriverRestartConfiguration.CONF
      .set(DriverRestartConfiguration.ON_DRIVER_RESTARTED,
          ReefEventStateManager.DriverRestartHandler.class)
      .set(DriverRestartConfiguration.ON_DRIVER_RESTART_TASK_RUNNING,
          ReefEventStateManager.DriverRestartTaskRunningStateHandler.class)
      .set(DriverRestartConfiguration.ON_DRIVER_RESTART_CONTEXT_ACTIVE,
          ReefEventStateManager.DriverRestartActiveContextStateHandler.class)
      .build();

  return Configurations.merge(yarnDriverRestartConfiguration, driverRestartHandlerConfigurations);
}
 

@Override
public Configuration getDriverConfiguration(final URI jobFolder,
                                            final String clientRemoteId,
                                            final String jobId,
                                            final Configuration applicationConfiguration) {
  return Configurations.merge(
    org.apache.reef.runtime.yarn.driver.YarnDriverConfiguration.CONF
            .set(JOB_SUBMISSION_DIRECTORY, jobFolder.toString())
            .set(JOB_IDENTIFIER, jobId)
            .set(CLIENT_REMOTE_IDENTIFIER, clientRemoteId)
            .set(JVM_HEAP_SLACK, this.jvmSlack)
            .set(RUNTIME_NAMES, RuntimeIdentifier.RUNTIME_NAME)
            .build(),
            applicationConfiguration);
}
 

/**
 * Builds the configuration.
 * @return The built configuration
 */
public Configuration build() {
  Validate.notNull(this.submissionRuntime, "Default Runtime was not defined");

  if(!this.defaultRuntime.isPresent() || this.runtimeNames.size() == 1){
    this.defaultRuntime = Optional.of(this.runtimeNames.toArray(new String[0])[0]);
  }

  Validate.isTrue(this.defaultRuntime.isPresent(),
          "Default runtime was not defined, and multiple runtimes were specified");

  if(!this.runtimeNames.contains(this.defaultRuntime.get())){
    this.runtimeNames.add(this.defaultRuntime.get());
  }

  JavaConfigurationBuilder conf = Tang.Factory.getTang().newConfigurationBuilder();

  for(Map.Entry<Class, Object> entry: this.namedParameters.entrySet()){
    conf = conf.bindNamedParameter(entry.getKey(), entry.getValue().toString());
  }

  conf = conf.bindNamedParameter(DefaultRuntimeName.class, this.defaultRuntime.get());

  for(final String runtimeName : this.runtimeNames){
    conf = conf.bindSetEntry(RuntimeNames.class, runtimeName);
  }

  if(!this.submissionRuntime.equalsIgnoreCase(RuntimeIdentifier.RUNTIME_NAME)){
    throw new RuntimeException("Unsupported submission runtime " + this.submissionRuntime);
  }

  conf = conf.bindImplementation(MultiRuntimeMainConfigurationGenerator.class,
          YarnMultiRuntimeMainConfigurationGeneratorImpl.class);

  // Currently only local runtime is supported as a secondary runtime
  return Configurations.merge(conf.build(),
          ExtensibleYarnClientConfiguration.CONF
                  .set(ExtensibleYarnClientConfiguration.DRIVER_CONFIGURATION_PROVIDER,
                          MultiRuntimeDriverConfigurationProvider.class).build());
}
 

@Before
public void setUp() throws InjectionException, IOException, ServletException {
  final Configuration clockConfiguration = HttpHandlerConfiguration.CONF
      .set(HttpHandlerConfiguration.HTTP_HANDLERS, HttpServerReefEventHandler.class)
      .build();
  final Configuration remoteConfiguration = Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(RemoteConfiguration.ManagerName.class, "REEF_TEST_REMOTE_MANAGER")
      .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
      .bindNamedParameter(JobIdentifier.class, "my job")
      .build();
  this.configuration = Configurations.merge(clockConfiguration, remoteConfiguration);
}
 

@Before
public void setUp() throws InjectionException, IOException, ServletException {

  this.request = new Request(
      new HttpConnection(new LocalConnector(), new StringEndPoint(), new Server()));

  this.request.setContentType("text/json");

  this.response = new Response(
      new HttpConnection(new LocalConnector(), new StringEndPoint(), new Server()));

  final Configuration httpHandlerConfiguration = HttpHandlerConfiguration.CONF
      .set(HttpHandlerConfiguration.HTTP_HANDLERS, HttpServerReefEventHandler.class)
      .build();

  final Tang tang = Tang.Factory.getTang();

  final Configuration remoteConfiguration = tang.newConfigurationBuilder()
      .bindNamedParameter(RemoteConfiguration.ManagerName.class, "REEF_TEST_REMOTE_MANAGER")
      .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
      .bindNamedParameter(JobIdentifier.class, "my job")
      .build();

  final Configuration finalConfig =
      Configurations.merge(httpHandlerConfiguration, remoteConfiguration);

  final Injector injector = tang.newInjector(finalConfig);

  this.handler = injector.getInstance(JettyHandler.class);
}
 

@Test
public void httpServerAddHandlerTest() throws Exception {

  final Configuration httpRuntimeConfiguration = HttpRuntimeConfiguration.CONF.build();

  final Configuration reefConfiguration = Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(RemoteConfiguration.ManagerName.class, "REEF_TEST_REMOTE_MANAGER")
      .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
      .bindNamedParameter(JobIdentifier.class, "my job")
      .build();

  final Configuration finalConfig =
      Configurations.merge(httpRuntimeConfiguration, reefConfiguration);

  final Injector injector = Tang.Factory.getTang().newInjector(finalConfig);
  final HttpServer httpServer = injector.getInstance(HttpServer.class);
  final HttpServerReefEventHandler httpHandler =
      injector.getInstance(HttpServerReefEventHandler.class);

  httpServer.addHttpHandler(httpHandler);

  // Assert.assertTrue(true);
  // Cannot access private variables inside the server.
  // If it is returned, meaning it is added successfully.

  httpServer.stop();
}
 

@Test
public void httpServerPortRetryTestWithTcpPortProvider() throws Exception {

  final Configuration httpRuntimeConfiguration = HttpRuntimeConfiguration.CONF.build();
  final Injector injector1 = Tang.Factory.getTang().newInjector(httpRuntimeConfiguration);
  final HttpServer httpServer1 = injector1.getInstance(HttpServer.class);
  final String portUsed = "" + httpServer1.getPort();

  final Configuration httpServerConfiguration = Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(TcpPortRangeCount.class, "1")
      .bindNamedParameter(TcpPortRangeBegin.class, portUsed)
      .bindNamedParameter(TcpPortRangeTryCount.class, "3")
      .build();

  final Configuration configuration =
      Configurations.merge(httpRuntimeConfiguration, httpServerConfiguration);

  final Injector injector2 = Tang.Factory.getTang().newInjector(configuration);

  try {
    injector2.getInstance(HttpServer.class);
    Assert.fail("Created two web servers on the same port: " + portUsed);
  } catch (final InjectionException ex) {
    Assert.assertEquals("Could not find available port for http", ex.getCause().getMessage());
  }

  httpServer1.stop();
}
 

public Configuration getRuntimeConfiguration(
    final ClientProtocol.DriverClientConfiguration driverClientConfiguration) {
  Configuration yarnConfiguration = YarnClientConfiguration.CONF
      .set(YarnClientConfiguration.UNMANAGED_DRIVER,
          driverClientConfiguration.getYarnRuntime().getUnmangedDriver())
      .set(YarnClientConfiguration.YARN_PRIORITY, driverClientConfiguration.getYarnRuntime().getPriority())
      .set(YarnClientConfiguration.JVM_HEAP_SLACK, 0.0)
      .build();
  if (StringUtils.isNotEmpty(driverClientConfiguration.getYarnRuntime().getFilesystemUrl())) {
    final JavaConfigurationBuilder providerConfig = Tang.Factory.getTang().newConfigurationBuilder()
        .bindNamedParameter(FileSystemUrl.class, driverClientConfiguration.getYarnRuntime().getFilesystemUrl());
    yarnConfiguration = Configurations.merge(yarnConfiguration, providerConfig.build());
  }
  return yarnConfiguration;
}
 

@Before
public void setUp() throws InjectionException {

  final Configuration httpHandlerConfiguration = HttpHandlerConfiguration.CONF
      .set(HttpHandlerConfiguration.HTTP_HANDLERS, HttpServerReefEventHandler.class)
      .build();

  final Configuration driverConfigurationForHttpServer = DriverServiceConfiguration.CONF
      .set(DriverServiceConfiguration.ON_EVALUATOR_ALLOCATED,
          ReefEventStateManager.AllocatedEvaluatorStateHandler.class)
      .set(DriverServiceConfiguration.ON_CONTEXT_ACTIVE, ReefEventStateManager.ActiveContextStateHandler.class)
      .set(DriverServiceConfiguration.ON_TASK_RUNNING, ReefEventStateManager.TaskRunningStateHandler.class)
      .set(DriverServiceConfiguration.ON_DRIVER_STARTED, ReefEventStateManager.StartStateHandler.class)
      .set(DriverServiceConfiguration.ON_DRIVER_STOP, ReefEventStateManager.StopStateHandler.class)
      .build();

  final Configuration contextConfig = Tang.Factory.getTang().newConfigurationBuilder()
      .bindImplementation(ActiveContext.class, MockActiveContext.class)
      .bindNamedParameter(RemoteConfiguration.ManagerName.class, "REEF_TEST_REMOTE_MANAGER")
      .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
      .bindNamedParameter(JobIdentifier.class, "my job")
      .build();

  final Configuration configuration = Configurations.merge(
      httpHandlerConfiguration, driverConfigurationForHttpServer, contextConfig);

  this.injector = Tang.Factory.getTang().newInjector(configuration);
}
 

/**
 * Assembles the Driver configuration.
 *
 * @param jobFolder the job folder.
 * @param clientRemoteId the client remote id.
 * @param jobId the job id.
 * @param applicationConfiguration the application configuration.
 * @return the Driver configuration.
 */
@Override
public Configuration getDriverConfiguration(final URI jobFolder,
                                            final String clientRemoteId,
                                            final String jobId,
                                            final Configuration applicationConfiguration) {
  ConfigurationModuleBuilder driverConfigurationBuilder = AzureBatchDriverConfiguration.CONF.getBuilder()
      .bindImplementation(CommandBuilder.class, this.commandBuilder.getClass());

  // If using docker containers, then use a different set of bindings
  if (this.containerRegistryProvider.isValid()) {
    driverConfigurationBuilder = driverConfigurationBuilder
        .bindImplementation(LocalAddressProvider.class, ContainerBasedLocalAddressProvider.class)
        .bindImplementation(TcpPortProvider.class, SetTcpPortProvider.class);
  }

  final Configuration driverConfiguration = driverConfigurationBuilder.build()
      .set(AzureBatchDriverConfiguration.JOB_IDENTIFIER, jobId)
      .set(AzureBatchDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId)
      .set(AzureBatchDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack)
      .set(AzureBatchDriverConfiguration.RUNTIME_NAME, RuntimeIdentifier.RUNTIME_NAME)
      .set(AzureBatchDriverConfiguration.AZURE_BATCH_ACCOUNT_URI, this.azureBatchAccountUri)
      .set(AzureBatchDriverConfiguration.AZURE_BATCH_ACCOUNT_NAME, this.azureBatchAccountName)
      .set(AzureBatchDriverConfiguration.AZURE_BATCH_POOL_ID, this.azureBatchPoolId)
      .set(AzureBatchDriverConfiguration.AZURE_STORAGE_ACCOUNT_NAME, this.azureStorageAccountName)
      .set(AzureBatchDriverConfiguration.AZURE_STORAGE_CONTAINER_NAME, this.azureStorageContainerName)
      .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_SERVER,
          this.containerRegistryProvider.getContainerRegistryServer())
      .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_USERNAME,
          this.containerRegistryProvider.getContainerRegistryUsername())
      .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_PASSWORD,
          this.containerRegistryProvider.getContainerRegistryPassword())
      .set(AzureBatchDriverConfiguration.CONTAINER_IMAGE_NAME,
          this.containerRegistryProvider.getContainerImageName())
      .setMultiple(AzureBatchDriverConfiguration.TCP_PORT_SET, this.tcpPortSet)
      .build();
  return Configurations.merge(driverConfiguration, applicationConfiguration);
}