Java Code Examples for org.apache.reef.tang.Configurations#merge()

The following examples show how to use org.apache.reef.tang.Configurations#merge() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NemoDriver.java    From nemo with Apache License 2.0 6 votes vote down vote up
private Configuration getExecutorConfiguration(final String executorId, final int executorCapacity) {
  final Configuration executorConfiguration = JobConf.EXECUTOR_CONF
      .set(JobConf.EXECUTOR_ID, executorId)
      .set(JobConf.EXECUTOR_CAPACITY, executorCapacity)
      .set(JobConf.GLUSTER_DISK_DIRECTORY, glusterDirectory)
      .set(JobConf.LOCAL_DISK_DIRECTORY, localDirectory)
      .set(JobConf.JOB_ID, jobId)
      .build();

  final Configuration contextConfiguration = ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, executorId) // We set: contextId = executorId
      .set(ContextConfiguration.ON_CONTEXT_STARTED, NemoContext.ContextStartHandler.class)
      .set(ContextConfiguration.ON_CONTEXT_STOP, NemoContext.ContextStopHandler.class)
      .build();

  final Configuration ncsConfiguration =  getExecutorNcsConfiguration();
  final Configuration messageConfiguration = getExecutorMessageConfiguration(executorId);

  return Configurations.merge(executorConfiguration, contextConfiguration, ncsConfiguration, messageConfiguration);
}
 
Example 2
Source File: HDInsightDriverConfigurationProviderImpl.java    From reef with Apache License 2.0 6 votes vote down vote up
@Override
public Configuration getDriverConfiguration(final URI jobFolder,
                                            final String clientRemoteId,
                                            final String jobId,
                                            final Configuration applicationConfiguration) {

  final Configuration hdinsightDriverConfiguration = HDInsightDriverConfiguration.CONF
          .set(HDInsightDriverConfiguration.JOB_IDENTIFIER, jobId)
          .set(HDInsightDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId)
          .set(HDInsightDriverConfiguration.JOB_SUBMISSION_DIRECTORY, jobFolder.toString())
          .set(HDInsightDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack)
          .set(HDInsightDriverConfiguration.RUNTIME_NAMES, RuntimeIdentifier.RUNTIME_NAME)
          .build();

  return Configurations.merge(
          applicationConfiguration,
          hdinsightDriverConfiguration);
}
 
Example 3
Source File: JobClient.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * @return the driver-side configuration to be merged into the DriverConfiguration to enable the HTTP server.
 */
public static Configuration getHTTPConfiguration() {
  final Configuration httpHandlerConfiguration = HttpHandlerConfiguration.CONF
      .set(HttpHandlerConfiguration.HTTP_HANDLERS, HttpServerReefEventHandler.class)
      .build();

  final Configuration driverConfigurationForHttpServer = DriverServiceConfiguration.CONF
      .set(DriverServiceConfiguration.ON_EVALUATOR_ALLOCATED,
          ReefEventStateManager.AllocatedEvaluatorStateHandler.class)
      .set(DriverServiceConfiguration.ON_CONTEXT_ACTIVE, ReefEventStateManager.ActiveContextStateHandler.class)
      .set(DriverServiceConfiguration.ON_TASK_RUNNING, ReefEventStateManager.TaskRunningStateHandler.class)
      .set(DriverServiceConfiguration.ON_DRIVER_STARTED, ReefEventStateManager.StartStateHandler.class)
      .set(DriverServiceConfiguration.ON_DRIVER_STOP, ReefEventStateManager.StopStateHandler.class)
      .build();

  return Configurations.merge(httpHandlerConfiguration, driverConfigurationForHttpServer);
}
 
Example 4
Source File: VortexJobConf.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * Instantiate a {@link VortexJobConf} object, where a Configuration is built by Tang internally.
 *
 * {@link IllegalArgumentException} will be thrown if required parameters are not set
 * (See {@link #setJobName(String)} and {@link #setVortexMasterConf(Configuration)}).
 *
 * Also, {@link org.apache.reef.tang.exceptions.BindException} can be thrown while merging the configurations.
 *
 * @return An instance of VortexJobConf object.
 */
@Override
public VortexJobConf build() {
  BuilderUtils.notNull(jobName);
  BuilderUtils.notNull(vortexMasterConf);

  final Configuration vortexDriverConf = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(VortexDriver.class))
      .set(DriverConfiguration.ON_DRIVER_STARTED, VortexDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, VortexDriver.AllocatedEvaluatorHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, VortexDriver.RunningTaskHandler.class)
      .set(DriverConfiguration.ON_TASK_MESSAGE, VortexDriver.TaskMessageHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_FAILED, VortexDriver.FailedEvaluatorHandler.class)
      .set(DriverConfiguration.DRIVER_IDENTIFIER, jobName)
      .build();

  final Configuration jobConf;
  if (userConf.isPresent()) {
    jobConf = Configurations.merge(vortexDriverConf, vortexMasterConf, userConf.get());
  } else {
    jobConf = Configurations.merge(vortexDriverConf, vortexMasterConf);
  }
  return new VortexJobConf(jobConf);
}
 
Example 5
Source File: HelloREEFHttp.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * @return the driver-side configuration to be merged into the DriverConfiguration to enable the HTTP server.
 */
public static Configuration getHTTPConfiguration() {
  final Configuration httpHandlerConfiguration = HttpHandlerConfiguration.CONF
      .set(HttpHandlerConfiguration.HTTP_HANDLERS, HttpServerReefEventHandler.class)
      .set(HttpHandlerConfiguration.HTTP_HANDLERS, HttpServerShellCmdHandler.class)
      .build();
  final Configuration driverConfigurationForHttpServer = DriverServiceConfiguration.CONF
      .set(DriverServiceConfiguration.ON_EVALUATOR_ALLOCATED,
          ReefEventStateManager.AllocatedEvaluatorStateHandler.class)
      .set(DriverServiceConfiguration.ON_CONTEXT_ACTIVE, ReefEventStateManager.ActiveContextStateHandler.class)
      .set(DriverServiceConfiguration.ON_TASK_RUNNING, ReefEventStateManager.TaskRunningStateHandler.class)
      .set(DriverServiceConfiguration.ON_DRIVER_STARTED, ReefEventStateManager.StartStateHandler.class)
      .set(DriverServiceConfiguration.ON_DRIVER_STOP, ReefEventStateManager.StopStateHandler.class)
      .build();
  return Configurations.merge(httpHandlerConfiguration, driverConfigurationForHttpServer);
}
 
Example 6
Source File: HelloREEFYarnRestart.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * @return the configuration of the HelloREEF driver.
 */
private static Configuration getDriverConfiguration() {
  return
      Configurations.merge(DriverConfiguration.CONF
              .set(DriverConfiguration.GLOBAL_LIBRARIES,
                  HelloREEFYarnRestart.class.getProtectionDomain().getCodeSource().getLocation().getFile())
              .set(DriverConfiguration.DRIVER_IDENTIFIER, "HelloREEF")
              .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, HelloDriver.EvaluatorAllocatedHandler.class)
              .build(),
          YarnDriverRestartConfiguration.CONF
              .build(),
          DriverRestartConfiguration.CONF
              .set(DriverRestartConfiguration.ON_DRIVER_RESTARTED,
                  HelloDriverRestart.DriverRestartHandler.class)
              .build());
}
 
Example 7
Source File: Scheduler.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * Submit a task to the ActiveContext.
 */
public synchronized void submitTask(final ActiveContext context) {
  final TaskEntity task = taskQueue.poll();
  final Integer taskId = task.getId();
  final String command = task.getCommand();

  final Configuration taskConf = TaskConfiguration.CONF
      .set(TaskConfiguration.TASK, ShellTask.class)
      .set(TaskConfiguration.IDENTIFIER, taskId.toString())
      .build();
  final Configuration commandConf = Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(Command.class, command)
      .build();

  final Configuration merged = Configurations.merge(taskConf, commandConf);
  context.submitTask(merged);
  runningTasks.add(task);
}
 
Example 8
Source File: TestHttpServer.java    From reef with Apache License 2.0 6 votes vote down vote up
@Test
public void httpServerSpecifiedPortTest() throws Exception {

  final Configuration httpRuntimeConfiguration = HttpRuntimeConfiguration.CONF.build();

  final Configuration httpServerConfiguration = Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(TcpPortRangeBegin.class, "9000")
      .build();

  final Configuration configuration =
      Configurations.merge(httpRuntimeConfiguration, httpServerConfiguration);

  final Injector injector = Tang.Factory.getTang().newInjector(configuration);
  final HttpServer httpServer = injector.getInstance(HttpServer.class);
  Assert.assertNotNull(httpServer);
  httpServer.stop();
}
 
Example 9
Source File: TestHttpServer.java    From reef with Apache License 2.0 5 votes vote down vote up
@Test
public void httpServerPortRetryTestWithTcpPortProvider() throws Exception {

  final Configuration httpRuntimeConfiguration = HttpRuntimeConfiguration.CONF.build();
  final Injector injector1 = Tang.Factory.getTang().newInjector(httpRuntimeConfiguration);
  final HttpServer httpServer1 = injector1.getInstance(HttpServer.class);
  final String portUsed = "" + httpServer1.getPort();

  final Configuration httpServerConfiguration = Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(TcpPortRangeCount.class, "1")
      .bindNamedParameter(TcpPortRangeBegin.class, portUsed)
      .bindNamedParameter(TcpPortRangeTryCount.class, "3")
      .build();

  final Configuration configuration =
      Configurations.merge(httpRuntimeConfiguration, httpServerConfiguration);

  final Injector injector2 = Tang.Factory.getTang().newInjector(configuration);

  try {
    injector2.getInstance(HttpServer.class);
    Assert.fail("Created two web servers on the same port: " + portUsed);
  } catch (final InjectionException ex) {
    Assert.assertEquals("Could not find available port for http", ex.getCause().getMessage());
  }

  httpServer1.stop();
}
 
Example 10
Source File: SchedulerREEF.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Run the Task scheduler. If '-retain true' option is passed via command line,
 * the scheduler reuses evaluators to submit new Tasks.
 * @param runtimeConf The runtime configuration (e.g. Local, YARN, etc)
 * @param args Command line arguments.
 * @throws InjectionException
 * @throws java.io.IOException
 */
public static void runTaskScheduler(final Configuration runtimeConf, final String[] args)
    throws InjectionException, IOException, ParseException {
  final Tang tang = Tang.Factory.getTang();

  final Configuration commandLineConf = CommandLine.parseToConfiguration(args, Retain.class);

  // Merge the configurations to run Driver
  final Configuration driverConf = Configurations.merge(getDriverConf(), getHttpConf(), commandLineConf);

  final REEF reef = tang.newInjector(runtimeConf).getInstance(REEF.class);
  reef.submit(driverConf);
}
 
Example 11
Source File: TestRuntimeStartHandler.java    From reef with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws InjectionException, IOException, ServletException {
  final Configuration clockConfiguration = HttpHandlerConfiguration.CONF
      .set(HttpHandlerConfiguration.HTTP_HANDLERS, HttpServerReefEventHandler.class)
      .build();
  final Configuration remoteConfiguration = Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(RemoteConfiguration.ManagerName.class, "REEF_TEST_REMOTE_MANAGER")
      .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
      .bindNamedParameter(JobIdentifier.class, "my job")
      .build();
  this.configuration = Configurations.merge(clockConfiguration, remoteConfiguration);
}
 
Example 12
Source File: FileResourceTest.java    From reef with Apache License 2.0 5 votes vote down vote up
@Test
public void testDriverFiles() throws IOException, BindException, InjectionException {

  final Set<File> theFiles = getTempFiles(this.nFiles);
  final Configuration finalDriverConfiguration = Configurations.merge(
      getDriverConfiguration(theFiles), getTestDriverConfiguration(theFiles));

  final LauncherStatus status = DriverLauncher
      .getLauncher(this.testEnvironment.getRuntimeConfiguration())
      .run(finalDriverConfiguration, testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 
Example 13
Source File: ClientLauncher.java    From reef with Apache License 2.0 5 votes vote down vote up
private LauncherStatus launch(final int port,
    final ClientProtocol.DriverClientConfiguration driverClientConfigurationProto) {
  try {
    final Configuration runtimeConfiguration = Configurations.merge(
        getClientServiceConfiguration(port),
        this.runtimeConfigurationProvider.getRuntimeConfiguration(driverClientConfigurationProto));
    try (ClientService clientService = TANG.newInjector(runtimeConfiguration).getInstance(ClientService.class)) {
      return clientService.submit(
          this.driverServiceConfigurationProvider.getDriverServiceConfiguration(driverClientConfigurationProto));
    }
  } catch (final InjectionException ex) {
    LOG.log(Level.SEVERE, "Job configuration error", ex);
    throw new RuntimeException("Could not launch driver service", ex);
  }
}
 
Example 14
Source File: YarnConfigurationProvider.java    From reef with Apache License 2.0 5 votes vote down vote up
public Configuration getRuntimeConfiguration(
    final ClientProtocol.DriverClientConfiguration driverClientConfiguration) {
  Configuration yarnConfiguration = YarnClientConfiguration.CONF
      .set(YarnClientConfiguration.UNMANAGED_DRIVER,
          driverClientConfiguration.getYarnRuntime().getUnmangedDriver())
      .set(YarnClientConfiguration.YARN_PRIORITY, driverClientConfiguration.getYarnRuntime().getPriority())
      .set(YarnClientConfiguration.JVM_HEAP_SLACK, 0.0)
      .build();
  if (StringUtils.isNotEmpty(driverClientConfiguration.getYarnRuntime().getFilesystemUrl())) {
    final JavaConfigurationBuilder providerConfig = Tang.Factory.getTang().newConfigurationBuilder()
        .bindNamedParameter(FileSystemUrl.class, driverClientConfiguration.getYarnRuntime().getFilesystemUrl());
    yarnConfiguration = Configurations.merge(yarnConfiguration, providerConfig.build());
  }
  return yarnConfiguration;
}
 
Example 15
Source File: AzureBatchDriverConfigurationProviderImpl.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Assembles the Driver configuration.
 *
 * @param jobFolder the job folder.
 * @param clientRemoteId the client remote id.
 * @param jobId the job id.
 * @param applicationConfiguration the application configuration.
 * @return the Driver configuration.
 */
@Override
public Configuration getDriverConfiguration(final URI jobFolder,
                                            final String clientRemoteId,
                                            final String jobId,
                                            final Configuration applicationConfiguration) {
  ConfigurationModuleBuilder driverConfigurationBuilder = AzureBatchDriverConfiguration.CONF.getBuilder()
      .bindImplementation(CommandBuilder.class, this.commandBuilder.getClass());

  // If using docker containers, then use a different set of bindings
  if (this.containerRegistryProvider.isValid()) {
    driverConfigurationBuilder = driverConfigurationBuilder
        .bindImplementation(LocalAddressProvider.class, ContainerBasedLocalAddressProvider.class)
        .bindImplementation(TcpPortProvider.class, SetTcpPortProvider.class);
  }

  final Configuration driverConfiguration = driverConfigurationBuilder.build()
      .set(AzureBatchDriverConfiguration.JOB_IDENTIFIER, jobId)
      .set(AzureBatchDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId)
      .set(AzureBatchDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack)
      .set(AzureBatchDriverConfiguration.RUNTIME_NAME, RuntimeIdentifier.RUNTIME_NAME)
      .set(AzureBatchDriverConfiguration.AZURE_BATCH_ACCOUNT_URI, this.azureBatchAccountUri)
      .set(AzureBatchDriverConfiguration.AZURE_BATCH_ACCOUNT_NAME, this.azureBatchAccountName)
      .set(AzureBatchDriverConfiguration.AZURE_BATCH_POOL_ID, this.azureBatchPoolId)
      .set(AzureBatchDriverConfiguration.AZURE_STORAGE_ACCOUNT_NAME, this.azureStorageAccountName)
      .set(AzureBatchDriverConfiguration.AZURE_STORAGE_CONTAINER_NAME, this.azureStorageContainerName)
      .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_SERVER,
          this.containerRegistryProvider.getContainerRegistryServer())
      .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_USERNAME,
          this.containerRegistryProvider.getContainerRegistryUsername())
      .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_PASSWORD,
          this.containerRegistryProvider.getContainerRegistryPassword())
      .set(AzureBatchDriverConfiguration.CONTAINER_IMAGE_NAME,
          this.containerRegistryProvider.getContainerImageName())
      .setMultiple(AzureBatchDriverConfiguration.TCP_PORT_SET, this.tcpPortSet)
      .build();
  return Configurations.merge(driverConfiguration, applicationConfiguration);
}
 
Example 16
Source File: JavaDriverClientLauncher.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * REEFLauncher is instantiated in the main() method below using
 * Tang configuration file provided as a command line argument.
 * @param configurationPath Path to the serialized Tang configuration file.
 * (The file must be in the local file system).
 * @param configurationSerializer Serializer used to read the configuration file.
 * We currently use Avro to serialize Tang configs.
 */
@Inject
private JavaDriverClientLauncher(
    @Parameter(DriverServicePort.class) final Integer driverServicePort,
    @Parameter(ClockConfigurationPath.class) final String configurationPath,
    final ConfigurationSerializer configurationSerializer) {

  this.envConfig = Configurations.merge(
      LAUNCHER_STATIC_CONFIG,
      DriverClientGrpcConfiguration.CONF
          .set(DriverClientGrpcConfiguration.DRIVER_SERVICE_PORT, driverServicePort)
          .build(),
      readConfigurationFromDisk(configurationPath, configurationSerializer));
}
 
Example 17
Source File: BGDClient.java    From reef with Apache License 2.0 4 votes vote down vote up
private Configuration getDriverConfiguration(final String jobName) {
  return Configurations.merge(
      getDataLoadConfiguration(jobName),
      GroupCommService.getConfiguration(fanOut),
      this.bgdControlParameters.getConfiguration());
}
 
Example 18
Source File: MultiRuntimeYarnBootstrapDriverConfigGenerator.java    From reef with Apache License 2.0 4 votes vote down vote up
private Configuration getMultiRuntimeDriverConfiguration(
        final AvroYarnJobSubmissionParameters yarnJobSubmissionParams,
        final AvroMultiRuntimeAppSubmissionParameters multiruntimeAppSubmissionParams) {

  if (multiruntimeAppSubmissionParams.getLocalRuntimeAppParameters() == null &&
          multiruntimeAppSubmissionParams.getYarnRuntimeAppParameters() == null){
    throw new IllegalArgumentException("At least on execution runtime has to be provided");
  }

  // read yarn job submission parameters
  final AvroJobSubmissionParameters jobSubmissionParameters =
          yarnJobSubmissionParams.getSharedJobSubmissionParameters();

  // generate multi runtime definition
  final MultiRuntimeDefinitionBuilder multiRuntimeDefinitionBuilder = new MultiRuntimeDefinitionBuilder();

  if (multiruntimeAppSubmissionParams.getLocalRuntimeAppParameters() != null){
    addLocalRuntimeDefinition(
            multiruntimeAppSubmissionParams.getLocalRuntimeAppParameters(),
            jobSubmissionParameters, multiRuntimeDefinitionBuilder);
  }

  if (multiruntimeAppSubmissionParams.getYarnRuntimeAppParameters() != null){
    addYarnRuntimeDefinition(
            yarnJobSubmissionParams,
            jobSubmissionParameters,
            multiRuntimeDefinitionBuilder);
  } else {
    addDummyYarnRuntimeDefinition(
            yarnJobSubmissionParams,
            jobSubmissionParameters,
            multiRuntimeDefinitionBuilder);
  }

  multiRuntimeDefinitionBuilder.setDefaultRuntimeName(
          multiruntimeAppSubmissionParams.getDefaultRuntimeName().toString());

  // generate multi runtime configuration
  ConfigurationModule multiRuntimeDriverConfiguration = MultiRuntimeDriverConfiguration.CONF
          .set(MultiRuntimeDriverConfiguration.JOB_IDENTIFIER, jobSubmissionParameters.getJobId().toString())
          .set(MultiRuntimeDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, ClientRemoteIdentifier.NONE)
          .set(MultiRuntimeDriverConfiguration.SERIALIZED_RUNTIME_DEFINITION,
                  this.runtimeDefinitionSerializer.toString(multiRuntimeDefinitionBuilder.build()));

  for (final CharSequence runtimeName : multiruntimeAppSubmissionParams.getRuntimes()){
    multiRuntimeDriverConfiguration = multiRuntimeDriverConfiguration.set(
            MultiRuntimeDriverConfiguration.RUNTIME_NAMES, runtimeName.toString());
  }

  final AvroAppSubmissionParameters appSubmissionParams =
          multiruntimeAppSubmissionParams.getSharedAppSubmissionParameters();

  // generate yarn related driver configuration
  final Configuration providerConfig = Tang.Factory.getTang().newConfigurationBuilder()
          .bindSetEntry(DriverConfigurationProviders.class, TcpPortConfigurationProvider.class)
          .bindNamedParameter(TcpPortRangeBegin.class, Integer.toString(appSubmissionParams.getTcpBeginPort()))
          .bindNamedParameter(TcpPortRangeCount.class, Integer.toString(appSubmissionParams.getTcpRangeCount()))
          .bindNamedParameter(TcpPortRangeTryCount.class, Integer.toString(appSubmissionParams.getTcpTryCount()))
          .bindNamedParameter(JobSubmissionDirectoryPrefix.class,
                  yarnJobSubmissionParams.getJobSubmissionDirectoryPrefix().toString())
          .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class)
          .bindConstructor(YarnConfiguration.class, YarnConfigurationConstructor.class)
          .build();

  final Configuration driverConfiguration = Configurations.merge(
          Constants.DRIVER_CONFIGURATION_WITH_HTTP_AND_NAMESERVER,
          multiRuntimeDriverConfiguration.build(),
          providerConfig);

  // add restart configuration if needed
  if (multiruntimeAppSubmissionParams.getYarnRuntimeAppParameters() != null &&
          multiruntimeAppSubmissionParams.getYarnRuntimeAppParameters().getDriverRecoveryTimeout() > 0) {
    LOG.log(Level.FINE, "Driver restart is enabled.");

    final Configuration yarnDriverRestartConfiguration =
            YarnDriverRestartConfiguration.CONF.build();

    final Configuration driverRestartConfiguration =
            DriverRestartConfiguration.CONF
                    .set(DriverRestartConfiguration.ON_DRIVER_RESTARTED, JobDriver.RestartHandler.class)
                    .set(DriverRestartConfiguration.ON_DRIVER_RESTART_CONTEXT_ACTIVE,
                            JobDriver.DriverRestartActiveContextHandler.class)
                    .set(DriverRestartConfiguration.ON_DRIVER_RESTART_TASK_RUNNING,
                            JobDriver.DriverRestartRunningTaskHandler.class)
                    .set(DriverRestartConfiguration.DRIVER_RESTART_EVALUATOR_RECOVERY_SECONDS,
                            multiruntimeAppSubmissionParams.getYarnRuntimeAppParameters().getDriverRecoveryTimeout())
                    .set(DriverRestartConfiguration.ON_DRIVER_RESTART_COMPLETED,
                            JobDriver.DriverRestartCompletedHandler.class)
                    .set(DriverRestartConfiguration.ON_DRIVER_RESTART_EVALUATOR_FAILED,
                            JobDriver.DriverRestartFailedEvaluatorHandler.class)
                    .build();

    return Configurations.merge(driverConfiguration, yarnDriverRestartConfiguration, driverRestartConfiguration);
  }

  return driverConfiguration;
}
 
Example 19
Source File: LocalDriverConfigurationProviderImpl.java    From reef with Apache License 2.0 3 votes vote down vote up
/**
 * Assembles the driver configuration.
 *
 * @param jobFolder                The folder in which the local runtime will execute this job.
 * @param clientRemoteId           the remote identifier of the client. It is used by the Driver to establish a
 *                                 connection back to the client.
 * @param jobId                    The identifier of the job.
 * @param applicationConfiguration The configuration of the application, e.g. a filled out DriverConfiguration
 * @return The Driver configuration to be used to instantiate the Driver.
 */
public Configuration getDriverConfiguration(final URI jobFolder,
                                            final String clientRemoteId,
                                            final String jobId,
                                            final Configuration applicationConfiguration) {
  return Configurations.merge(getDriverConfiguration(jobFolder, clientRemoteId, jobId), applicationConfiguration);
}
 
Example 20
Source File: HelloREEFHttp.java    From reef with Apache License 2.0 3 votes vote down vote up
/**
 * Run Hello Reef with merged configuration.
 *
 * @param runtimeConf
 * @param timeOut
 * @return
 * @throws BindException
 * @throws InjectionException
 */
public static LauncherStatus runHelloReef(final Configuration runtimeConf, final int timeOut)
    throws BindException, InjectionException {
  final Configuration driverConf =
      Configurations.merge(HelloREEFHttp.getDriverConfiguration(), getHTTPConfiguration());
  return DriverLauncher.getLauncher(runtimeConf).run(driverConf, timeOut);
}