com.google.appengine.tools.pipeline.JobSetting Java Examples

/**
 * Constructs a new JobRecord given the provided data. The constructed
 * instance will be inflated in the sense that
 * {@link #getJobInstanceInflated()}, {@link #getFinalizeBarrierInflated()},
 * {@link #getOutputSlotInflated()} and {@link #getRunBarrierInflated()} will
 * all not return {@code null}. This constructor is used when a new JobRecord
 * is created during the run() method of a parent job. The parent job is also
 * known as the generator job.
 *
 * @param generatorJob The parent generator job of this job.
 * @param graphGUIDParam The GUID of the local graph of this job.
 * @param jobInstance The non-null user-supplied instance of {@code Job} that
 *        implements the Job that the newly created JobRecord represents.
 * @param callExceptionHandler The flag that indicates that this job should call
 *        {@code Job#handleException(Throwable)} instead of {@code run}.
 * @param settings Array of {@code JobSetting} to apply to the newly created
 *        JobRecord.
 */
public JobRecord(JobRecord generatorJob, String graphGUIDParam, Job<?> jobInstance,
    boolean callExceptionHandler, JobSetting[] settings) {
  this(generatorJob.getRootJobKey(), null, generatorJob.getKey(), graphGUIDParam, jobInstance,
      callExceptionHandler, settings, generatorJob.getQueueSettings());
  // If generator job has exception handler then it should be called in case
  // of this job throwing to create an exception handling child job.
  // If callExceptionHandler is true then this job is an exception handling
  // child and its exceptions should be handled by its parent's
  // exceptionHandlingAncestor to avoid infinite recursion.
  if (generatorJob.isExceptionHandlerSpecified() && !callExceptionHandler) {
    exceptionHandlingAncestorKey = generatorJob.getKey();
  } else {
    exceptionHandlingAncestorKey = generatorJob.getExceptionHandlingAncestorKey();
  }
  // Inherit settings from generator job
  Map<Class<? extends JobSetting>, JobSetting> settingsMap = new HashMap<>();
  for (JobSetting setting : settings) {
    settingsMap.put(setting.getClass(), setting);
  }
  if (!settingsMap.containsKey(StatusConsoleUrl.class)) {
    statusConsoleUrl = generatorJob.statusConsoleUrl;
  }
}
 

/**
 * Creates and launches a new Pipeline
 * <p>
 * Creates the root Job with its associated Barriers and Slots and saves them
 * to the data store. All slots in the root job are immediately filled with
 * the values of the parameters to this method, and
 * {@link HandleSlotFilledTask HandleSlotFilledTasks} are enqueued for each of
 * the filled slots.
 *
 * @param settings JobSetting array used to control details of the Pipeline
 * @param jobInstance A user-supplied instance of {@link Job} that will serve
 *        as the root job of the Pipeline.
 * @param params Arguments to the root job's run() method
 * @return The pipelineID of the newly created pipeline, also known as the
 *         rootJobID.
 */
public static String startNewPipeline(
    JobSetting[] settings, Job<?> jobInstance, Object... params) {
  UpdateSpec updateSpec = new UpdateSpec(null);
  Job<?> rootJobInstance = jobInstance;
  // If rootJobInstance has exceptionHandler it has to be wrapped to ensure that root job
  // ends up in finalized state in case of exception of run method and
  // exceptionHandler returning a result.
  if (JobRecord.isExceptionHandlerSpecified(jobInstance)) {
    rootJobInstance = new RootJobInstance(jobInstance, settings, params);
    params = new Object[0];
  }
  // Create the root Job and its associated Barriers and Slots
  // Passing null for parent JobRecord and graphGUID
  // Create HandleSlotFilledTasks for the input parameters.
  JobRecord jobRecord = registerNewJobRecord(
      updateSpec, settings, null, null, rootJobInstance, params);
  updateSpec.setRootJobKey(jobRecord.getRootJobKey());
  // Save the Pipeline model objects and enqueue the tasks that start the Pipeline executing.
  backEnd.save(updateSpec, jobRecord.getQueueSettings());
  return jobRecord.getKey().getName();
}
 

private MapreduceRunnerResult runAsPipeline(Job0<?> job) {
  String jobId =
      newPipelineService()
          .startNewPipeline(
              job, new JobSetting.OnModule(moduleName), new JobSetting.OnQueue(QUEUE_NAME));
  logger.atInfo().log(
      "Started '%s' %s job: %s",
      jobName, job instanceof MapJob ? "map" : "mapreduce", renderMapreduceConsoleLink(jobId));
  return new MapreduceRunnerResult(jobId);
}
 

/**
 * @param updateSpec UpdateSpec to use
 * @param jobRecord record of the job with exception handler to execute
 * @param caughtException failure cause
 * @param ignoreException if failure should be ignored (used for cancellation)
 */
private static void executeExceptionHandler(UpdateSpec updateSpec, JobRecord jobRecord,
    Throwable caughtException, boolean ignoreException) {
  updateSpec.getNonTransactionalGroup().includeJob(jobRecord);
  String errorHandlingGraphGuid = GUIDGenerator.nextGUID();
  Job<?> jobInstance = jobRecord.getJobInstanceInflated().getJobInstanceDeserialized();

  JobRecord errorHandlingJobRecord =
      new JobRecord(jobRecord, errorHandlingGraphGuid, jobInstance, true, new JobSetting[0]);
  errorHandlingJobRecord.setOutputSlotInflated(jobRecord.getOutputSlotInflated());
  errorHandlingJobRecord.setIgnoreException(ignoreException);
  registerNewJobRecord(updateSpec, errorHandlingJobRecord,
      new Object[] {new ImmediateValue<>(caughtException)});
}
 

private void applySetting(JobSetting setting) {
  if (setting instanceof WaitForSetting) {
    WaitForSetting wf = (WaitForSetting) setting;
    FutureValueImpl<?> fv = (FutureValueImpl<?>) wf.getValue();
    Slot slot = fv.getSlot();
    runBarrierInflated.addPhantomArgumentSlot(slot);
  } else if (setting instanceof IntValuedSetting) {
    int value = ((IntValuedSetting) setting).getValue();
    if (setting instanceof BackoffSeconds) {
      backoffSeconds = value;
    } else if (setting instanceof BackoffFactor) {
      backoffFactor = value;
    } else if (setting instanceof MaxAttempts) {
      maxAttempts = value;
    } else {
      throw new RuntimeException("Unrecognized JobOption class " + setting.getClass().getName());
    }
  } else if (setting instanceof OnBackend) {
    queueSettings.setOnBackend(((OnBackend) setting).getValue());
  } else if (setting instanceof OnModule) {
    queueSettings.setOnModule(((OnModule) setting).getValue());
  } else if (setting instanceof OnQueue) {
    queueSettings.setOnQueue(((OnQueue) setting).getValue());
  } else if (setting instanceof StatusConsoleUrl){
    statusConsoleUrl = ((StatusConsoleUrl) setting).getValue();
  } else {
    throw new RuntimeException("Unrecognized JobOption class " + setting.getClass().getName());
  }
}
 

@Override
public <T1, T2, T3, T4, T5, T6> String startNewPipeline(
    Job6<?, T1, T2, T3, T4, T5, T6> jobInstance, T1 arg1, T2 arg2, T3 arg3, T4 arg4, T5 arg5,
    T6 arg6, JobSetting... settings) {
  return PipelineManager.startNewPipeline(settings, jobInstance, arg1, arg2, arg3, arg4, arg5,
      arg6);
}
 

private Value<Void> execute() {
    JobSetting.WaitForSetting waitClearSinks = waitFor(futureCall(new DrainSinksJob(), immediate(pipeClazz), null));
    return futureCall(new FlushSourcesJob(), immediate(pipeClazz), null, waitClearSinks);
}
 

public RootJobInstance(Job<?> jobInstance, JobSetting[] settings, Object[] params) {
  this.jobInstance = jobInstance;
  this.settings = settings;
  this.params = params;
}
 

@Override
public String startNewPipelineUnchecked(Job<?> jobInstance, Object[] arguments,
    JobSetting... settings) {
  return PipelineManager.startNewPipeline(settings, jobInstance, arguments);
}
 

@Override
public <T1, T2, T3, T4, T5> String startNewPipeline(Job5<?, T1, T2, T3, T4, T5> jobInstance,
    T1 arg1, T2 arg2, T3 arg3, T4 arg4, T5 arg5, JobSetting... settings) {
  return PipelineManager.startNewPipeline(settings, jobInstance, arg1, arg2, arg3, arg4, arg5);
}
 

@Override
public <T1, T2, T3, T4> String startNewPipeline(Job4<?, T1, T2, T3, T4> jobInstance, T1 arg1,
    T2 arg2, T3 arg3, T4 arg4, JobSetting... settings) {
  return PipelineManager.startNewPipeline(settings, jobInstance, arg1, arg2, arg3, arg4);
}
 

@Override
public <T1, T2, T3> String startNewPipeline(Job3<?, T1, T2, T3> jobInstance, T1 arg1, T2 arg2,
    T3 arg3, JobSetting... settings) {
  return PipelineManager.startNewPipeline(settings, jobInstance, arg1, arg2, arg3);
}
 

@Override
public <T1, T2> String startNewPipeline(Job2<?, T1, T2> jobInstance, T1 arg1, T2 arg2,
    JobSetting... settings) {
  return PipelineManager.startNewPipeline(settings, jobInstance, arg1, arg2);
}
 

@Override
public <T1> String startNewPipeline(Job1<?, T1> jobInstance, T1 arg1, JobSetting... settings) {
  return PipelineManager.startNewPipeline(settings, jobInstance, arg1);
}
 

@Override
public String startNewPipeline(Job0<?> jobInstance, JobSetting... settings) {
  return PipelineManager.startNewPipeline(settings, jobInstance);
}
 

private JobRecord(Key key, Job<?> jobInstance, JobSetting[] settings) {
  // Root Jobs have their rootJobKey the same as their keys and provide null for generatorKey
  // and graphGUID. Also, callExceptionHandler is always false.
  this(key, key, null, null, jobInstance, false, settings, null);
  rootJobDisplayName = jobInstance.getJobDisplayName();
}
 

/**
 * Creates a new JobRecord with its associated Barriers and Slots. Also
 * creates new {@link HandleSlotFilledTask} for any inputs to the Job that are
 * immediately specified. Registers all newly created objects with the
 * provided {@code UpdateSpec} for later saving.
 * <p>
 * This method is called when starting a new Pipeline, in which case it is
 * used to create the root job, and it is called from within the run() method
 * of a generator job in order to create a child job.
 *
 * @param updateSpec The {@code UpdateSpec} with which to register all newly
 *        created objects. All objects will be added to the
 *        {@link UpdateSpec#getNonTransactionalGroup() non-transaction group}
 *        of the {@code UpdateSpec}.
 * @param settings Array of {@code JobSetting} to apply to the newly created
 *        JobRecord.
 * @param generatorJob The generator job or {@code null} if we are creating
 *        the root job.
 * @param graphGUID The GUID of the child graph to which the new Job belongs
 *        or {@code null} if we are creating the root job.
 * @param jobInstance The user-supplied instance of {@code Job} that
 *        implements the Job that the newly created JobRecord represents.
 * @param params The arguments to be passed to the run() method of the newly
 *        created Job. Each argument may be an actual value or it may be an
 *        object of type {@link Value} representing either an
 *        {@link ImmediateValue} or a
 *        {@link com.google.appengine.tools.pipeline.FutureValue FutureValue}.
 *        For each element of the array, if the Object is not of type
 *        {@link Value} then it is interpreted as an {@link ImmediateValue}
 *        with the given Object as its value.
 * @return The newly constructed JobRecord.
 */
public static JobRecord registerNewJobRecord(UpdateSpec updateSpec, JobSetting[] settings,
    JobRecord generatorJob, String graphGUID, Job<?> jobInstance, Object[] params) {
  JobRecord jobRecord;
  if (generatorJob == null) {
    // Root Job
    if (graphGUID != null) {
      throw new IllegalArgumentException("graphGUID must be null for root jobs");
    }
    jobRecord = JobRecord.createRootJobRecord(jobInstance, settings);
  } else {
    jobRecord = new JobRecord(generatorJob, graphGUID, jobInstance, false, settings);
  }
  return registerNewJobRecord(updateSpec, jobRecord, params);
}
 

/**
 * A factory method for root jobs.
 *
 * @param jobInstance The non-null user-supplied instance of {@code Job} that
 *        implements the Job that the newly created JobRecord represents.
 * @param settings Array of {@code JobSetting} to apply to the newly created
 *        JobRecord.
 */
public static JobRecord createRootJobRecord(Job<?> jobInstance, JobSetting[] settings) {
  Key key = generateKey(null, DATA_STORE_KIND);
  return new JobRecord(key, jobInstance, settings);
}