org.apache.reef.wake.time.event.StartTime Java Examples

@Override
public void onNext(final StartTime startTime) {
  HelloMultiRuntimeDriver.this.requestor.newRequest()
      .setNumber(1)
      .setMemory(64)
      .setNumberOfCores(1)
      .setRuntimeName(RuntimeIdentifier.RUNTIME_NAME)
      .submit();

  LOG.log(Level.INFO, "Requested Local Evaluator .");

  HelloMultiRuntimeDriver.this.requestor.newRequest()
      .setNumber(1)
      .setMemory(64)
      .setNumberOfCores(1)
      .setRuntimeName(RuntimeIdentifier.RUNTIME_NAME)
      .submit();

  LOG.log(Level.INFO, "Requested Yarn Evaluator.");
}
 

@Override
public void onNext(final StartTime startTime) {
  LOG.log(Level.INFO, "StartTime: {0} Number of files in the set: {1}",
      new Object[]{startTime, FileResourceTestDriver.this.fileNamesToExpect.size()});

  // Check whether the files made it
  for (final String fileName : FileResourceTestDriver.this.fileNamesToExpect) {
    final File file = new File(localFolder, fileName);
    LOG.log(Level.INFO, "Testing file: " + file.getAbsolutePath());
    if (!file.exists()) {
      throw new DriverSideFailure("Cannot find file: " + fileName);
    } else if (!file.isFile()) {
      throw new DriverSideFailure("Not a file: " + fileName);
    } else if (!file.canRead()) {
      throw new DriverSideFailure("Can't read: " + fileName);
    }
  }

  // Ask for a single evaluator.
  FileResourceTestDriver.this.requestor.submit(EvaluatorRequest.newBuilder()
      .setNumber(1).setMemory(64).setNumberOfCores(1).build());
}
 

@Override
public void driverRestartHandler(final DriverRestartInfo request, final StreamObserver<Void> responseObserver) {
  LOG.log(Level.INFO, "Driver restarted");
  try (ObserverCleanup cleanup = ObserverCleanup.of(responseObserver)) {
    final DriverRestarted driverRestarted = new DriverRestarted() {
      @Override
      public int getResubmissionAttempts() {
        return request.getResubmissionAttempts();
      }

      @Override
      public StartTime getStartTime() {
        return new StartTime(request.getStartTime().getStartTime());
      }

      @Override
      public Set<String> getExpectedEvaluatorIds() {
        return new HashSet<>(request.getExpectedEvaluatorIdsList());
      }
    };
    this.clientDriverDispatcher.get().dispatchRestart(driverRestarted);
  }
}
 

@Inject
private RuntimeClock(
    final Timer timer,
    @Parameter(Clock.StartHandler.class)
        final InjectionFuture<Set<EventHandler<StartTime>>> startHandler,
    @Parameter(Clock.StopHandler.class)
        final InjectionFuture<Set<EventHandler<StopTime>>> stopHandler,
    @Parameter(Clock.RuntimeStartHandler.class)
        final InjectionFuture<Set<EventHandler<RuntimeStart>>> runtimeStartHandler,
    @Parameter(Clock.RuntimeStopHandler.class)
        final InjectionFuture<Set<EventHandler<RuntimeStop>>> runtimeStopHandler,
    @Parameter(Clock.IdleHandler.class)
        final InjectionFuture<Set<EventHandler<IdleClock>>> idleHandler) {

  this.timer = timer;
  this.startHandler = startHandler;
  this.stopHandler = stopHandler;
  this.runtimeStartHandler = runtimeStartHandler;
  this.runtimeStopHandler = runtimeStopHandler;
  this.idleHandler = idleHandler;

  LOG.log(Level.FINE, "RuntimeClock instantiated.");
}
 

@Inject
private DriverStartHandler(
    @Parameter(org.apache.reef.driver.parameters.DriverStartHandler.class)
      final Set<EventHandler<StartTime>> startHandlers,
    @Parameter(DriverRestartHandler.class)
      final Set<EventHandler<DriverRestarted>> restartHandlers,
    @Parameter(ServiceDriverRestartedHandlers.class)
      final Set<EventHandler<DriverRestarted>> serviceRestartHandlers,
    final DriverRestartManager driverRestartManager) {

  this.startHandlers = startHandlers;
  this.restartHandlers = restartHandlers;
  this.serviceRestartHandlers = serviceRestartHandlers;
  this.driverRestartManager = driverRestartManager;

  LOG.log(Level.FINE,
      "Instantiated DriverStartHandler: StartHandlers:{0} RestartHandlers:{1} ServiceRestartHandlers:{2}",
      new Object[] {this.startHandlers, this.restartHandlers, this.serviceRestartHandlers});
}
 

private void onRestart(final StartTime startTime) {

    if (this.restartHandlers.isEmpty()) {
      throw new DriverFatalRuntimeException("Driver restarted, but no ON_DRIVER_RESTART handler is bound.");
    }

    final List<EventHandler<DriverRestarted>> orderedRestartHandlers =
        new ArrayList<>(this.serviceRestartHandlers.size() + this.restartHandlers.size());

    orderedRestartHandlers.addAll(this.serviceRestartHandlers);
    orderedRestartHandlers.addAll(this.restartHandlers);

    // This can only be called after calling client restart handlers because REEF.NET
    // JobDriver requires making this call to set up the InterOp handlers.
    this.driverRestartManager.onRestart(startTime, orderedRestartHandlers);
  }
 

@Override
public void startHandler(final StartTime startTime) {
  try {
    start();
  } catch (final IOException | InterruptedException e) {
    throw new RuntimeException("unable to start driver client", e);
  }
  synchronized (this) {
    if (this.clientStub != null) {
      this.clientStub.startHandler(
          StartTimeInfo.newBuilder().setStartTime(startTime.getTimestamp()).build());
    } else if (!stopped) {
      stop(new RuntimeException("Unable to start driver client"));
    }
  }
}
 

/**
 * Generate a DriverRestarted event to be passed to the
 * {@link org.apache.reef.driver.parameters.DriverRestartHandler}.
 * @return DriverRestarted event based on the state at the time of driver failure
 */
public DriverRestarted getDriverRestarted() {
  final Set<String> expectedEvaluatorIds = new HashSet<>();
  for (final MockAllocatedEvaluator allocatedEvaluator : this.allocatedEvaluators) {
    expectedEvaluatorIds.add(allocatedEvaluator.getId());
  }
  return new DriverRestarted() {
    @Override
    public int getResubmissionAttempts() {
      return restartAttemps;
    }

    @Override
    public StartTime getStartTime() {
      return startTime;
    }

    @Override
    public Set<String> getExpectedEvaluatorIds() {
      return expectedEvaluatorIds;
    }
  };
}
 

@Test
@PrepareForTest({HeronReefUtils.class, SchedulerMain.class})
public void onNextStartTimeStartsSchedulerTMaster() throws Exception {
  PowerMockito.spy(HeronReefUtils.class);
  PowerMockito.doNothing().when(HeronReefUtils.class,
      "extractPackageInSandbox",
      anyString(),
      anyString(),
      anyString());

  SchedulerMain mockScheduler = mock(SchedulerMain.class);
  PowerMockito.spy(SchedulerMain.class);
  PowerMockito.doReturn(mockScheduler).when(SchedulerMain.class,
      "createInstance",
      anyString(),
      anyString(),
      anyString(),
      anyString(),
      anyString(),
      eq(0),
      eq(false));

  spyDriver.new HeronSchedulerLauncher().onNext(new StartTime(System.currentTimeMillis()));

  verify(mockScheduler, times(1)).runScheduler();
}
 

public MockDriverRestartContext(
    final int restartAttemps,
    final StartTime startTime,
    final List<MockAllocatedEvaluator> allocatedEvaluators,
    final List<MockActiveContext> activeContexts,
    final List<MockRunningTask> runningTasks) {
  this.restartAttemps = restartAttemps;
  this.startTime = startTime;
  this.allocatedEvaluators = allocatedEvaluators;
  this.activeContexts = activeContexts;
  this.runningTasks = runningTasks;
  this.failedEvaluators = new ArrayList<>();
}
 

/** StartTime event: launch another REEF job. */
@Override
public void onNext(final StartTime startTime) {

  LOG.log(Level.INFO, "Driver started: app {0} :: {1}", new Object[] {this.hostApplicationId, startTime});
  LOG.log(Level.FINE, "Launching Unmanaged AM: {0}", JAR_PATH);

  try (DriverLauncher client = DriverLauncher.getLauncher(RUNTIME_CONFIG)) {

    final String innerApplicationId = client.submit(DRIVER_CONFIG, 10000);
    LOG.log(Level.INFO, "REEF-on-REEF job submitted: host app {0} new app {1}",
        new Object[] {this.hostApplicationId, innerApplicationId});

    final Configuration yarnAmConfig = UnmanagedAmYarnDriverConfiguration.CONF
        .set(UnmanagedAmYarnDriverConfiguration.JOB_IDENTIFIER, innerApplicationId)
        .set(UnmanagedAmYarnDriverConfiguration.JOB_SUBMISSION_DIRECTORY, DRIVER_ROOT_PATH)
        .build();

    try (REEFEnvironment reef =
        REEFEnvironment.fromConfiguration(client.getUser(), yarnAmConfig, DRIVER_CONFIG)) {

      reef.run();

      final ReefServiceProtos.JobStatusProto status = reef.getLastStatus();

      LOG.log(Level.INFO, "REEF-on-REEF inner job {0} completed: state {1}",
          new Object[] {innerApplicationId, status.getState()});
    }

    LOG.log(Level.INFO,
        "REEF-on-REEF host job {0} completed: inner app {1} status {2}",
        new Object[] {this.hostApplicationId, innerApplicationId, client.getStatus()});

  } catch (final InjectionException ex) {
    LOG.log(Level.SEVERE, "REEF-on-REEF configuration error", ex);
    throw new RuntimeException("REEF-on-REEF configuration error", ex);
  }

  ThreadLogger.logThreads(LOG, Level.FINEST, "Threads running after DriverLauncher.close():");
}
 

@Override
public void onNext(final StartTime startTime) {
  LOG.log(Level.FINE, "Request {0} Evaluators.", numEvaluatorsToSubmit);
  FailureDriver.this.requestor.submit(EvaluatorRequest.newBuilder()
      .setNumber(numEvaluatorsToSubmit)
      .setMemory(64)
      .setNumberOfCores(1)
      .build());
}
 

@Override
public void onNext(final StartTime startTime) {
  requestor.submit(EvaluatorRequest.newBuilder()
      .setNumber(4)
      .setMemory(128)
      .build());
}
 

@Override
public void onNext(final StartTime value) {
  evaluatorRequestor.submit(EvaluatorRequest
      .newBuilder()
      .setMemory(64)
      .setNumberOfCores(1)
      .setNumber(2)
      .build());
}
 

@Override
public void onNext(final StartTime startTime) {
  evaluatorRequestor.submit(EvaluatorRequest.newBuilder()
      .setNumber(2)
      .setMemory(128)
      .build());
}
 

@Override
public void onNext(final StartTime startTime) {
  // Initial Evaluator Request
  evaluatorRequestor.submit(EvaluatorRequest.newBuilder()
      .setNumber(evalNum)
      .setMemory(evalMem)
      .setNumberOfCores(evalCores)
      .build());

  // Run Vortex Start
  vortexStartEStage.onNext(vortexStart);

  // Run Scheduler
  pendingTaskletSchedulerEStage.onNext(SCHEDULER_EVENT);
}
 

@Override
public void onNext(final StartTime startTime) {
  EvaluatorSizeTestDriver.this.evaluatorRequestor.submit(EvaluatorRequest.newBuilder()
      .setNumber(1)
      .setMemory(EvaluatorSizeTestDriver.this.memorySize)
      .setNumberOfCores(1)
      .build());
}
 

@Override
public void startHandler(final StartTimeInfo request, final StreamObserver<Void> responseObserver) {
  try (ObserverCleanup cleanup = ObserverCleanup.of(responseObserver)) {
    LOG.log(Level.INFO, "StartHandler at time {0}", request.getStartTime());
    final StartTime startTime = new StartTime(request.getStartTime());
    this.clientDriverDispatcher.get().dispatch(startTime);
  }
}
 

@Override
public void onNext(final StartTime startTime) {
  synchronized (HttpShellJobDriver.this) {
    LOG.log(Level.INFO, "{0} StartTime: {1}", new Object[]{state, startTime});
    assert state == State.INIT;
    requestEvaluators();
  }
}
 

/**
 * Recovers the list of alive and failed evaluators and inform the driver restart handlers and inform the
 * evaluator failure handlers based on the specific runtime. Also sets the expected amount of evaluators to report
 * back as alive to the job driver.
 */
public synchronized void onRestart(final StartTime startTime,
                                   final List<EventHandler<DriverRestarted>> orderedHandlers) {
  if (this.state == DriverRestartState.BEGAN) {
    restartEvaluators = driverRuntimeRestartManager.getPreviousEvaluators();
    final DriverRestarted restartedInfo = new DriverRestartedImpl(resubmissionAttempts, startTime, restartEvaluators);

    for (final EventHandler<DriverRestarted> handler : orderedHandlers) {
      handler.onNext(restartedInfo);
    }

    this.state = DriverRestartState.IN_PROGRESS;
  } else {
    final String errMsg = "Should not be setting the set of expected alive evaluators more than once.";
    LOG.log(Level.SEVERE, errMsg);
    throw new DriverFatalRuntimeException(errMsg);
  }

  driverRuntimeRestartManager.informAboutEvaluatorFailures(getFailedEvaluators());

  if (driverRestartEvaluatorRecoverySeconds != Integer.MAX_VALUE) {
    // Don't use Clock here because if there is an event scheduled, the driver will not be idle, even if
    // driver restart has already completed, and we cannot cancel the event.
    restartCompletedTimer.schedule(new TimerTask() {
      @Override
      public void run() {
        onDriverRestartCompleted(true);
      }
    }, driverRestartEvaluatorRecoverySeconds * 1000L);
  }
}
 

DriverRestartedImpl(final int resubmissionAttempts,
                    final StartTime startTime,
                    final RestartEvaluators restartEvaluators) {
  this.resubmissionAttempts = resubmissionAttempts;
  this.startTime = startTime;
  final Set<String> expected = new HashSet<>();

  for (final String evaluatorId : restartEvaluators.getEvaluatorIds()) {
    if (restartEvaluators.get(evaluatorId).getEvaluatorRestartState() == EvaluatorRestartState.EXPECTED) {
      expected.add(evaluatorId);
    }
  }

  this.expectedEvaluatorIds = Collections.unmodifiableSet(expected);
}
 

@Override
public void onNext(final StartTime time) {
  LOG.log(Level.INFO, "StartTime: {0}", time);
  evaluatorRequestor.newRequest()
      .setMemory(128)
      .setNumberOfCores(1)
      .setNumber(NUM_EVALUATORS)
      .submit();
}
 

@Override
public void onNext(final StartTime startTime) {
  if (this.driverRestartManager.detectRestart()) {
    this.onRestart(startTime);
  } else {
    this.onStart(startTime);
  }
}
 

@Override
public void onNext(final StartTime startTime) {
  HelloJVMOptionsDriver.this.requestor.newRequest()
      .setNumber(1)
      .setMemory(64)
      .setNumberOfCores(1)
      .submit();
  LOG.log(Level.INFO, "Requested Evaluator.");
}
 

@Override
public void onNext(final StartTime startTime) {
  HelloDriver.this.requestor.newRequest()
      .setNumber(1)
      .setMemory(64)
      .setNumberOfCores(1)
      .submit();
  LOG.log(Level.INFO, "Requested Evaluator.");
}
 

@Override
public void onNext(final StartTime startTime) {
  LOG.log(Level.INFO, "TIME: Start Driver with {0} Evaluators", numEvaluators);
  evaluatorRequestor.newRequest()
      .setMemory(128)
      .setNumberOfCores(1)
      .setNumber(numEvaluators)
      .submit();
}
 

@Override
public void onNext(final StartTime startTime) {
  synchronized (SchedulerDriver.this) {
    LOG.log(Level.INFO, "Driver started at {0}", startTime);
    assert state == State.INIT;
    state = State.WAIT_EVALUATORS;

    requestEvaluator(1); // Allocate an initial evaluator to avoid idle state.
  }
}
 

@Override
public void onNext(final StartTime startTime) {
  requestor.submit(EvaluatorRequest.newBuilder()
      .setNumber(numEvaluators)
      .setMemory(64)
      .setNumberOfCores(1)
      .build());
}
 

@Override
public void onNext(final StartTime startTime) {
  OutputServiceDriver.this.requestor.newRequest()
      .setNumber(3)
      .setMemory(64)
      .setNumberOfCores(1)
      .submit();
  LOG.log(Level.INFO, "Requested Evaluator.");
}
 

@Override
public void onNext(final StartTime startTime) {
  final int numEvals = BroadcastDriver.this.numberOfReceivers + 1;
  LOG.log(Level.FINE, "Requesting {0} evaluators", numEvals);
  BroadcastDriver.this.requestor.newRequest()
      .setNumber(numEvals)
      .setMemory(2048)
      .submit();
}