Java Code Examples for org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager

The following examples show how to use org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestJobImpl.java    License: Apache License 2.0 6 votes vote down vote up
public StubbedJob(JobId jobId, ApplicationAttemptId applicationAttemptId,
    Configuration conf, EventHandler eventHandler, boolean newApiCommitter,
    String user, int numSplits, AppContext appContext) {
  super(jobId, applicationAttemptId, conf, eventHandler,
      null, new JobTokenSecretManager(), new Credentials(),
      new SystemClock(), Collections.<TaskId, TaskInfo> emptyMap(),
      MRAppMetrics.create(), null, newApiCommitter, user,
      System.currentTimeMillis(), null, appContext, null, null);

  initTransition = getInitTransition(numSplits);
  localFactory = stateMachineFactory.addTransition(JobStateInternal.NEW,
        EnumSet.of(JobStateInternal.INITED, JobStateInternal.FAILED),
        JobEventType.JOB_INIT,
        // This is abusive.
        initTransition);

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  localStateMachine = localFactory.make(this);
}
 
Example 2
Source Project: hadoop   Source File: MRApp.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
public TestJob(JobId jobId, ApplicationAttemptId applicationAttemptId,
    Configuration conf, EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener, Clock clock,
    OutputCommitter committer, boolean newApiCommitter,
    String user, AppContext appContext,
    JobStateInternal forcedState, String diagnostic) {
  super(jobId, getApplicationAttemptId(applicationId, getStartCount()),
      conf, eventHandler, taskAttemptListener,
      new JobTokenSecretManager(), new Credentials(), clock,
      getCompletedTaskFromPreviousRun(), metrics, committer,
      newApiCommitter, user, System.currentTimeMillis(), getAllAMInfos(),
      appContext, forcedState, diagnostic);

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  localStateMachine = localFactory.make(this);
}
 
Example 3
Source Project: hadoop   Source File: TestShuffleHandler.java    License: Apache License 2.0 6 votes vote down vote up
private static int getShuffleResponseCode(ShuffleHandler shuffle,
    Token<JobTokenIdentifier> jt) throws IOException {
  URL url = new URL("http://127.0.0.1:"
      + shuffle.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
      + "/mapOutput?job=job_12345_0001&reduce=0&map=attempt_12345_1_m_1_0");
  HttpURLConnection conn = (HttpURLConnection) url.openConnection();
  String encHash = SecureShuffleUtils.hashFromString(
      SecureShuffleUtils.buildMsgFrom(url),
      JobTokenSecretManager.createSecretKey(jt.getPassword()));
  conn.addRequestProperty(
      SecureShuffleUtils.HTTP_HEADER_URL_HASH, encHash);
  conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
      ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
  conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
      ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
  conn.connect();
  int rc = conn.getResponseCode();
  conn.disconnect();
  return rc;
}
 
Example 4
Source Project: hadoop   Source File: TestFetcher.java    License: Apache License 2.0 6 votes vote down vote up
@Before
@SuppressWarnings("unchecked") // mocked generics
public void setup() {
  LOG.info(">>>> " + name.getMethodName());
  job = new JobConf();
  job.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, false);
  jobWithRetry = new JobConf();
  jobWithRetry.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, true);
  id = TaskAttemptID.forName("attempt_0_1_r_1_1");
  ss = mock(ShuffleSchedulerImpl.class);
  mm = mock(MergeManagerImpl.class);
  r = mock(Reporter.class);
  metrics = mock(ShuffleClientMetrics.class);
  except = mock(ExceptionReporter.class);
  key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
  connection = mock(HttpURLConnection.class);

  allErrs = mock(Counters.Counter.class);
  when(r.getCounter(anyString(), anyString())).thenReturn(allErrs);

  ArrayList<TaskAttemptID> maps = new ArrayList<TaskAttemptID>(1);
  maps.add(map1ID);
  maps.add(map2ID);
  when(ss.getMapsForHost(host)).thenReturn(maps);
}
 
Example 5
Source Project: big-c   Source File: TestJobImpl.java    License: Apache License 2.0 6 votes vote down vote up
public StubbedJob(JobId jobId, ApplicationAttemptId applicationAttemptId,
    Configuration conf, EventHandler eventHandler, boolean newApiCommitter,
    String user, int numSplits, AppContext appContext) {
  super(jobId, applicationAttemptId, conf, eventHandler,
      null, new JobTokenSecretManager(), new Credentials(),
      new SystemClock(), Collections.<TaskId, TaskInfo> emptyMap(),
      MRAppMetrics.create(), null, newApiCommitter, user,
      System.currentTimeMillis(), null, appContext, null, null);

  initTransition = getInitTransition(numSplits);
  localFactory = stateMachineFactory.addTransition(JobStateInternal.NEW,
        EnumSet.of(JobStateInternal.INITED, JobStateInternal.FAILED),
        JobEventType.JOB_INIT,
        // This is abusive.
        initTransition);

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  localStateMachine = localFactory.make(this);
}
 
Example 6
Source Project: big-c   Source File: MRApp.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
public TestJob(JobId jobId, ApplicationAttemptId applicationAttemptId,
    Configuration conf, EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener, Clock clock,
    OutputCommitter committer, boolean newApiCommitter,
    String user, AppContext appContext,
    JobStateInternal forcedState, String diagnostic) {
  super(jobId, getApplicationAttemptId(applicationId, getStartCount()),
      conf, eventHandler, taskAttemptListener,
      new JobTokenSecretManager(), new Credentials(), clock,
      getCompletedTaskFromPreviousRun(), metrics, committer,
      newApiCommitter, user, System.currentTimeMillis(), getAllAMInfos(),
      appContext, forcedState, diagnostic);

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  localStateMachine = localFactory.make(this);
}
 
Example 7
Source Project: big-c   Source File: TestShuffleHandler.java    License: Apache License 2.0 6 votes vote down vote up
private static int getShuffleResponseCode(ShuffleHandler shuffle,
    Token<JobTokenIdentifier> jt) throws IOException {
  URL url = new URL("http://127.0.0.1:"
      + shuffle.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
      + "/mapOutput?job=job_12345_0001&reduce=0&map=attempt_12345_1_m_1_0");
  HttpURLConnection conn = (HttpURLConnection) url.openConnection();
  String encHash = SecureShuffleUtils.hashFromString(
      SecureShuffleUtils.buildMsgFrom(url),
      JobTokenSecretManager.createSecretKey(jt.getPassword()));
  conn.addRequestProperty(
      SecureShuffleUtils.HTTP_HEADER_URL_HASH, encHash);
  conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
      ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
  conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
      ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
  conn.connect();
  int rc = conn.getResponseCode();
  conn.disconnect();
  return rc;
}
 
Example 8
Source Project: big-c   Source File: TestFetcher.java    License: Apache License 2.0 6 votes vote down vote up
@Before
@SuppressWarnings("unchecked") // mocked generics
public void setup() {
  LOG.info(">>>> " + name.getMethodName());
  job = new JobConf();
  job.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, false);
  jobWithRetry = new JobConf();
  jobWithRetry.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, true);
  id = TaskAttemptID.forName("attempt_0_1_r_1_1");
  ss = mock(ShuffleSchedulerImpl.class);
  mm = mock(MergeManagerImpl.class);
  r = mock(Reporter.class);
  metrics = mock(ShuffleClientMetrics.class);
  except = mock(ExceptionReporter.class);
  key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
  connection = mock(HttpURLConnection.class);

  allErrs = mock(Counters.Counter.class);
  when(r.getCounter(anyString(), anyString())).thenReturn(allErrs);

  ArrayList<TaskAttemptID> maps = new ArrayList<TaskAttemptID>(1);
  maps.add(map1ID);
  maps.add(map2ID);
  when(ss.getMapsForHost(host)).thenReturn(maps);
}
 
Example 9
Source Project: hadoop   Source File: TaskAttemptListenerImpl.java    License: Apache License 2.0 5 votes vote down vote up
public TaskAttemptListenerImpl(AppContext context,
    JobTokenSecretManager jobTokenSecretManager,
    RMHeartbeatHandler rmHeartbeatHandler,
    byte[] secretShuffleKey) {
  super(TaskAttemptListenerImpl.class.getName());
  this.context = context;
  this.jobTokenSecretManager = jobTokenSecretManager;
  this.rmHeartbeatHandler = rmHeartbeatHandler;
  this.encryptedSpillKey = secretShuffleKey;
}
 
Example 10
Source Project: hadoop   Source File: TestTaskAttemptListenerImpl.java    License: Apache License 2.0 5 votes vote down vote up
public MockTaskAttemptListenerImpl(AppContext context,
    JobTokenSecretManager jobTokenSecretManager,
    RMHeartbeatHandler rmHeartbeatHandler,
    TaskHeartbeatHandler hbHandler) {
  super(context, jobTokenSecretManager, rmHeartbeatHandler, null);
  this.taskHeartbeatHandler = hbHandler;
}
 
Example 11
Source Project: hadoop   Source File: TestJobImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReportDiagnostics() throws Exception {
  JobID jobID = JobID.forName("job_1234567890000_0001");
  JobId jobId = TypeConverter.toYarn(jobID);
  final String diagMsg = "some diagnostic message";
  final JobDiagnosticsUpdateEvent diagUpdateEvent =
      new JobDiagnosticsUpdateEvent(jobId, diagMsg);
  MRAppMetrics mrAppMetrics = MRAppMetrics.create();
  AppContext mockContext = mock(AppContext.class);
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
  JobImpl job = new JobImpl(jobId, Records
      .newRecord(ApplicationAttemptId.class), new Configuration(),
      mock(EventHandler.class),
      null, mock(JobTokenSecretManager.class), null,
      new SystemClock(), null,
      mrAppMetrics, null, true, null, 0, null, mockContext, null, null);
  job.handle(diagUpdateEvent);
  String diagnostics = job.getReport().getDiagnostics();
  Assert.assertNotNull(diagnostics);
  Assert.assertTrue(diagnostics.contains(diagMsg));

  job = new JobImpl(jobId, Records
      .newRecord(ApplicationAttemptId.class), new Configuration(),
      mock(EventHandler.class),
      null, mock(JobTokenSecretManager.class), null,
      new SystemClock(), null,
      mrAppMetrics, null, true, null, 0, null, mockContext, null, null);
  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  job.handle(diagUpdateEvent);
  diagnostics = job.getReport().getDiagnostics();
  Assert.assertNotNull(diagnostics);
  Assert.assertTrue(diagnostics.contains(diagMsg));
}
 
Example 12
Source Project: hadoop   Source File: TestJobImpl.java    License: Apache License 2.0 5 votes vote down vote up
private boolean testUberDecision(Configuration conf) {
  JobID jobID = JobID.forName("job_1234567890000_0001");
  JobId jobId = TypeConverter.toYarn(jobID);
  MRAppMetrics mrAppMetrics = MRAppMetrics.create();
  JobImpl job =
      new JobImpl(jobId, ApplicationAttemptId.newInstance(
        ApplicationId.newInstance(0, 0), 0), conf, mock(EventHandler.class),
        null, new JobTokenSecretManager(), new Credentials(), null, null,
        mrAppMetrics, null, true, null, 0, null, null, null, null);
  InitTransition initTransition = getInitTransition(2);
  JobEvent mockJobEvent = mock(JobEvent.class);
  initTransition.transition(job, mockJobEvent);
  boolean isUber = job.isUber();
  return isUber;
}
 
Example 13
Source Project: hadoop   Source File: ShuffleHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  userRsrc = new ConcurrentHashMap<String,String>();
  secretManager = new JobTokenSecretManager();
  recoverState(conf);
  ServerBootstrap bootstrap = new ServerBootstrap(selector);
  try {
    pipelineFact = new HttpPipelineFactory(conf);
  } catch (Exception ex) {
    throw new RuntimeException(ex);
  }
  bootstrap.setPipelineFactory(pipelineFact);
  port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);
  Channel ch = bootstrap.bind(new InetSocketAddress(port));
  accepted.add(ch);
  port = ((InetSocketAddress)ch.getLocalAddress()).getPort();
  conf.set(SHUFFLE_PORT_CONFIG_KEY, Integer.toString(port));
  pipelineFact.SHUFFLE.setPort(port);
  LOG.info(getName() + " listening on port " + port);
  super.serviceStart();

  sslFileBufferSize = conf.getInt(SUFFLE_SSL_FILE_BUFFER_SIZE_KEY,
                                  DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE);
  connectionKeepAliveEnabled =
      conf.getBoolean(SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED,
        DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED);
  connectionKeepAliveTimeOut =
      Math.max(1, conf.getInt(SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT,
        DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT));
  mapOutputMetaInfoCacheSize =
      Math.max(1, conf.getInt(SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE,
        DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE));
}
 
Example 14
Source Project: big-c   Source File: TaskAttemptListenerImpl.java    License: Apache License 2.0 5 votes vote down vote up
public TaskAttemptListenerImpl(AppContext context,
    JobTokenSecretManager jobTokenSecretManager,
    RMHeartbeatHandler rmHeartbeatHandler,
    byte[] secretShuffleKey) {
  super(TaskAttemptListenerImpl.class.getName());
  this.context = context;
  this.jobTokenSecretManager = jobTokenSecretManager;
  this.rmHeartbeatHandler = rmHeartbeatHandler;
  this.encryptedSpillKey = secretShuffleKey;
}
 
Example 15
Source Project: big-c   Source File: TestTaskAttemptListenerImpl.java    License: Apache License 2.0 5 votes vote down vote up
public MockTaskAttemptListenerImpl(AppContext context,
    JobTokenSecretManager jobTokenSecretManager,
    RMHeartbeatHandler rmHeartbeatHandler,
    TaskHeartbeatHandler hbHandler) {
  super(context, jobTokenSecretManager, rmHeartbeatHandler, null);
  this.taskHeartbeatHandler = hbHandler;
}
 
Example 16
Source Project: big-c   Source File: TestJobImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReportDiagnostics() throws Exception {
  JobID jobID = JobID.forName("job_1234567890000_0001");
  JobId jobId = TypeConverter.toYarn(jobID);
  final String diagMsg = "some diagnostic message";
  final JobDiagnosticsUpdateEvent diagUpdateEvent =
      new JobDiagnosticsUpdateEvent(jobId, diagMsg);
  MRAppMetrics mrAppMetrics = MRAppMetrics.create();
  AppContext mockContext = mock(AppContext.class);
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
  JobImpl job = new JobImpl(jobId, Records
      .newRecord(ApplicationAttemptId.class), new Configuration(),
      mock(EventHandler.class),
      null, mock(JobTokenSecretManager.class), null,
      new SystemClock(), null,
      mrAppMetrics, null, true, null, 0, null, mockContext, null, null);
  job.handle(diagUpdateEvent);
  String diagnostics = job.getReport().getDiagnostics();
  Assert.assertNotNull(diagnostics);
  Assert.assertTrue(diagnostics.contains(diagMsg));

  job = new JobImpl(jobId, Records
      .newRecord(ApplicationAttemptId.class), new Configuration(),
      mock(EventHandler.class),
      null, mock(JobTokenSecretManager.class), null,
      new SystemClock(), null,
      mrAppMetrics, null, true, null, 0, null, mockContext, null, null);
  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  job.handle(diagUpdateEvent);
  diagnostics = job.getReport().getDiagnostics();
  Assert.assertNotNull(diagnostics);
  Assert.assertTrue(diagnostics.contains(diagMsg));
}
 
Example 17
Source Project: big-c   Source File: TestJobImpl.java    License: Apache License 2.0 5 votes vote down vote up
private boolean testUberDecision(Configuration conf) {
  JobID jobID = JobID.forName("job_1234567890000_0001");
  JobId jobId = TypeConverter.toYarn(jobID);
  MRAppMetrics mrAppMetrics = MRAppMetrics.create();
  JobImpl job =
      new JobImpl(jobId, ApplicationAttemptId.newInstance(
        ApplicationId.newInstance(0, 0), 0), conf, mock(EventHandler.class),
        null, new JobTokenSecretManager(), new Credentials(), null, null,
        mrAppMetrics, null, true, null, 0, null, null, null, null);
  InitTransition initTransition = getInitTransition(2);
  JobEvent mockJobEvent = mock(JobEvent.class);
  initTransition.transition(job, mockJobEvent);
  boolean isUber = job.isUber();
  return isUber;
}
 
Example 18
Source Project: big-c   Source File: ShuffleHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  userRsrc = new ConcurrentHashMap<String,String>();
  secretManager = new JobTokenSecretManager();
  recoverState(conf);
  ServerBootstrap bootstrap = new ServerBootstrap(selector);
  try {
    pipelineFact = new HttpPipelineFactory(conf);
  } catch (Exception ex) {
    throw new RuntimeException(ex);
  }
  bootstrap.setPipelineFactory(pipelineFact);
  port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);
  Channel ch = bootstrap.bind(new InetSocketAddress(port));
  accepted.add(ch);
  port = ((InetSocketAddress)ch.getLocalAddress()).getPort();
  conf.set(SHUFFLE_PORT_CONFIG_KEY, Integer.toString(port));
  pipelineFact.SHUFFLE.setPort(port);
  LOG.info(getName() + " listening on port " + port);
  super.serviceStart();

  sslFileBufferSize = conf.getInt(SUFFLE_SSL_FILE_BUFFER_SIZE_KEY,
                                  DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE);
  connectionKeepAliveEnabled =
      conf.getBoolean(SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED,
        DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED);
  connectionKeepAliveTimeOut =
      Math.max(1, conf.getInt(SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT,
        DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT));
  mapOutputMetaInfoCacheSize =
      Math.max(1, conf.getInt(SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE,
        DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE));
}
 
Example 19
Source Project: incubator-tez   Source File: MRTask.java    License: Apache License 2.0 5 votes vote down vote up
private void configureMRTask()
    throws IOException, InterruptedException {

  Credentials credentials = UserGroupInformation.getCurrentUser()
      .getCredentials();
  jobConf.setCredentials(credentials);
  // TODO Can this be avoided all together. Have the MRTezOutputCommitter use
  // the Tez parameter.
  // TODO This could be fetched from the env if YARN is setting it for all
  // Containers.
  // Set it in conf, so as to be able to be used the the OutputCommitter.

  // Not needed. This is probably being set via the source/consumer meta
  Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
  if (jobToken != null) {
    // Will MR ever run without a job token.
    SecretKey sk = JobTokenSecretManager.createSecretKey(jobToken
        .getPassword());
    this.jobTokenSecret = sk;
  } else {
    LOG.warn("No job token set");
  }

  configureLocalDirs();

  // Set up the DistributedCache related configs
  setupDistributedCacheConfig(jobConf);
}
 
Example 20
Source Project: tez   Source File: MRTask.java    License: Apache License 2.0 5 votes vote down vote up
private void configureMRTask()
    throws IOException, InterruptedException {

  Credentials credentials = UserGroupInformation.getCurrentUser()
      .getCredentials();
  jobConf.setCredentials(credentials);
  // TODO Can this be avoided all together. Have the MRTezOutputCommitter use
  // the Tez parameter.
  // TODO This could be fetched from the env if YARN is setting it for all
  // Containers.
  // Set it in conf, so as to be able to be used the the OutputCommitter.

  // Not needed. This is probably being set via the source/consumer meta
  Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
  if (jobToken != null) {
    // Will MR ever run without a job token.
    SecretKey sk = JobTokenSecretManager.createSecretKey(jobToken
        .getPassword());
    this.jobTokenSecret = sk;
  } else {
    LOG.warn("No job token set");
  }

  configureLocalDirs();

  // Set up the DistributedCache related configs
  setupDistributedCacheConfig(jobConf);
}
 
Example 21
Source Project: hadoop   Source File: YarnChild.java    License: Apache License 2.0 4 votes vote down vote up
private static void configureTask(JobConf job, Task task,
    Credentials credentials, Token<JobTokenIdentifier> jt) throws IOException {
  job.setCredentials(credentials);
  
  ApplicationAttemptId appAttemptId =
      ConverterUtils.toContainerId(
          System.getenv(Environment.CONTAINER_ID.name()))
          .getApplicationAttemptId();
  LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptId);
  // Set it in conf, so as to be able to be used the the OutputCommitter.
  job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
      appAttemptId.getAttemptId());

  // set tcp nodelay
  job.setBoolean("ipc.client.tcpnodelay", true);
  job.setClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS,
      YarnOutputFiles.class, MapOutputFile.class);
  // set the jobToken and shuffle secrets into task
  task.setJobTokenSecret(
      JobTokenSecretManager.createSecretKey(jt.getPassword()));
  byte[] shuffleSecret = TokenCache.getShuffleSecretKey(credentials);
  if (shuffleSecret == null) {
    LOG.warn("Shuffle secret missing from task credentials."
        + " Using job token secret as shuffle secret.");
    shuffleSecret = jt.getPassword();
  }
  task.setShuffleSecret(
      JobTokenSecretManager.createSecretKey(shuffleSecret));

  // setup the child's MRConfig.LOCAL_DIR.
  configureLocalDirs(task, job);

  // setup the child's attempt directories
  // Do the task-type specific localization
  task.localizeConfiguration(job);

  // Set up the DistributedCache related configs
  MRApps.setupDistributedCacheLocal(job);

  // Overwrite the localized task jobconf which is linked to in the current
  // work-dir.
  Path localTaskFile = new Path(MRJobConfig.JOB_CONF_FILE);
  writeLocalJobFile(localTaskFile, job);
  task.setJobFile(localTaskFile.toString());
  task.setConf(job);
}
 
Example 22
Source Project: hadoop   Source File: JobImpl.java    License: Apache License 2.0 4 votes vote down vote up
public JobImpl(JobId jobId, ApplicationAttemptId applicationAttemptId,
    Configuration conf, EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener,
    JobTokenSecretManager jobTokenSecretManager,
    Credentials jobCredentials, Clock clock,
    Map<TaskId, TaskInfo> completedTasksFromPreviousRun, MRAppMetrics metrics,
    OutputCommitter committer, boolean newApiCommitter, String userName,
    long appSubmitTime, List<AMInfo> amInfos, AppContext appContext,
    JobStateInternal forcedState, String forcedDiagnostic) {
  this.applicationAttemptId = applicationAttemptId;
  this.jobId = jobId;
  this.jobName = conf.get(JobContext.JOB_NAME, "<missing job name>");
  this.conf = new JobConf(conf);
  this.metrics = metrics;
  this.clock = clock;
  this.completedTasksFromPreviousRun = completedTasksFromPreviousRun;
  this.amInfos = amInfos;
  this.appContext = appContext;
  this.userName = userName;
  this.queueName = conf.get(MRJobConfig.QUEUE_NAME, "default");
  this.appSubmitTime = appSubmitTime;
  this.oldJobId = TypeConverter.fromYarn(jobId);
  this.committer = committer;
  this.newApiCommitter = newApiCommitter;

  this.taskAttemptListener = taskAttemptListener;
  this.eventHandler = eventHandler;
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  this.readLock = readWriteLock.readLock();
  this.writeLock = readWriteLock.writeLock();

  this.jobCredentials = jobCredentials;
  this.jobTokenSecretManager = jobTokenSecretManager;

  this.aclsManager = new JobACLsManager(conf);
  this.username = System.getProperty("user.name");
  this.jobACLs = aclsManager.constructJobACLs(conf);

  ThreadFactory threadFactory = new ThreadFactoryBuilder()
    .setNameFormat("Job Fail Wait Timeout Monitor #%d")
    .setDaemon(true)
    .build();
  this.executor = new ScheduledThreadPoolExecutor(1, threadFactory);

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  stateMachine = stateMachineFactory.make(this);
  this.forcedState  = forcedState;
  if(forcedDiagnostic != null) {
    this.diagnostics.add(forcedDiagnostic);
  }
  
  this.maxAllowedFetchFailuresFraction = conf.getFloat(
      MRJobConfig.MAX_ALLOWED_FETCH_FAILURES_FRACTION,
      MRJobConfig.DEFAULT_MAX_ALLOWED_FETCH_FAILURES_FRACTION);
  this.maxFetchFailuresNotifications = conf.getInt(
      MRJobConfig.MAX_FETCH_FAILURES_NOTIFICATIONS,
      MRJobConfig.DEFAULT_MAX_FETCH_FAILURES_NOTIFICATIONS);
}
 
Example 23
Source Project: hadoop   Source File: TestTaskAttemptListenerImpl.java    License: Apache License 2.0 4 votes vote down vote up
public MockTaskAttemptListenerImpl(AppContext context,
    JobTokenSecretManager jobTokenSecretManager,
    RMHeartbeatHandler rmHeartbeatHandler) {
  super(context, jobTokenSecretManager, rmHeartbeatHandler, null);
}
 
Example 24
Source Project: hadoop   Source File: TestTaskAttemptListenerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Test  (timeout=5000)
public void testGetTask() throws IOException {
  AppContext appCtx = mock(AppContext.class);
  JobTokenSecretManager secret = mock(JobTokenSecretManager.class); 
  RMHeartbeatHandler rmHeartbeatHandler =
      mock(RMHeartbeatHandler.class);
  TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class);
  MockTaskAttemptListenerImpl listener = 
    new MockTaskAttemptListenerImpl(appCtx, secret,
        rmHeartbeatHandler, hbHandler);
  Configuration conf = new Configuration();
  listener.init(conf);
  listener.start();
  JVMId id = new JVMId("foo",1, true, 1);
  WrappedJvmID wid = new WrappedJvmID(id.getJobId(), id.isMap, id.getId());

  // Verify ask before registration.
  //The JVM ID has not been registered yet so we should kill it.
  JvmContext context = new JvmContext();
  context.jvmId = id; 
  JvmTask result = listener.getTask(context);
  assertNotNull(result);
  assertTrue(result.shouldDie);

  // Verify ask after registration but before launch. 
  // Don't kill, should be null.
  TaskAttemptId attemptID = mock(TaskAttemptId.class);
  Task task = mock(Task.class);
  //Now put a task with the ID
  listener.registerPendingTask(task, wid);
  result = listener.getTask(context);
  assertNull(result);
  // Unregister for more testing.
  listener.unregister(attemptID, wid);

  // Verify ask after registration and launch
  //Now put a task with the ID
  listener.registerPendingTask(task, wid);
  listener.registerLaunchedTask(attemptID, wid);
  verify(hbHandler).register(attemptID);
  result = listener.getTask(context);
  assertNotNull(result);
  assertFalse(result.shouldDie);
  // Don't unregister yet for more testing.

  //Verify that if we call it again a second time we are told to die.
  result = listener.getTask(context);
  assertNotNull(result);
  assertTrue(result.shouldDie);

  listener.unregister(attemptID, wid);

  // Verify after unregistration.
  result = listener.getTask(context);
  assertNotNull(result);
  assertTrue(result.shouldDie);

  listener.stop();

  // test JVMID
  JVMId jvmid = JVMId.forName("jvm_001_002_m_004");
  assertNotNull(jvmid);
  try {
    JVMId.forName("jvm_001_002_m_004_006");
    Assert.fail();
  } catch (IllegalArgumentException e) {
    assertEquals(e.getMessage(),
        "TaskId string : jvm_001_002_m_004_006 is not properly formed");
  }

}
 
Example 25
Source Project: hadoop   Source File: Application.java    License: Apache License 2.0 4 votes vote down vote up
public static String createDigest(byte[] password, String data)
    throws IOException {
  SecretKey key = JobTokenSecretManager.createSecretKey(password);
  return SecureShuffleUtils.hashFromString(data, key);
}
 
Example 26
Source Project: big-c   Source File: YarnChild.java    License: Apache License 2.0 4 votes vote down vote up
private static void configureTask(JobConf job, Task task,
    Credentials credentials, Token<JobTokenIdentifier> jt) throws IOException {
  job.setCredentials(credentials);
  
  ApplicationAttemptId appAttemptId =
      ConverterUtils.toContainerId(
          System.getenv(Environment.CONTAINER_ID.name()))
          .getApplicationAttemptId();
  LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptId);
  // Set it in conf, so as to be able to be used the the OutputCommitter.
  job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
      appAttemptId.getAttemptId());

  // set tcp nodelay
  job.setBoolean("ipc.client.tcpnodelay", true);
  job.setClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS,
      YarnOutputFiles.class, MapOutputFile.class);
  // set the jobToken and shuffle secrets into task
  task.setJobTokenSecret(
      JobTokenSecretManager.createSecretKey(jt.getPassword()));
  byte[] shuffleSecret = TokenCache.getShuffleSecretKey(credentials);
  if (shuffleSecret == null) {
    LOG.warn("Shuffle secret missing from task credentials."
        + " Using job token secret as shuffle secret.");
    shuffleSecret = jt.getPassword();
  }
  task.setShuffleSecret(
      JobTokenSecretManager.createSecretKey(shuffleSecret));

  // setup the child's MRConfig.LOCAL_DIR.
  configureLocalDirs(task, job);

  // setup the child's attempt directories
  // Do the task-type specific localization
  task.localizeConfiguration(job);

  // Set up the DistributedCache related configs
  MRApps.setupDistributedCacheLocal(job);

  // Overwrite the localized task jobconf which is linked to in the current
  // work-dir.
  Path localTaskFile = new Path(MRJobConfig.JOB_CONF_FILE);
  writeLocalJobFile(localTaskFile, job);
  task.setJobFile(localTaskFile.toString());
  task.setConf(job);
}
 
Example 27
Source Project: big-c   Source File: JobImpl.java    License: Apache License 2.0 4 votes vote down vote up
public JobImpl(JobId jobId, ApplicationAttemptId applicationAttemptId,
    Configuration conf, EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener,
    JobTokenSecretManager jobTokenSecretManager,
    Credentials jobCredentials, Clock clock,
    Map<TaskId, TaskInfo> completedTasksFromPreviousRun, MRAppMetrics metrics,
    OutputCommitter committer, boolean newApiCommitter, String userName,
    long appSubmitTime, List<AMInfo> amInfos, AppContext appContext,
    JobStateInternal forcedState, String forcedDiagnostic) {
  this.applicationAttemptId = applicationAttemptId;
  this.jobId = jobId;
  this.jobName = conf.get(JobContext.JOB_NAME, "<missing job name>");
  this.conf = new JobConf(conf);
  this.metrics = metrics;
  this.clock = clock;
  this.completedTasksFromPreviousRun = completedTasksFromPreviousRun;
  this.amInfos = amInfos;
  this.appContext = appContext;
  this.userName = userName;
  this.queueName = conf.get(MRJobConfig.QUEUE_NAME, "default");
  this.appSubmitTime = appSubmitTime;
  this.oldJobId = TypeConverter.fromYarn(jobId);
  this.committer = committer;
  this.newApiCommitter = newApiCommitter;

  this.taskAttemptListener = taskAttemptListener;
  this.eventHandler = eventHandler;
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  this.readLock = readWriteLock.readLock();
  this.writeLock = readWriteLock.writeLock();

  this.jobCredentials = jobCredentials;
  this.jobTokenSecretManager = jobTokenSecretManager;

  this.aclsManager = new JobACLsManager(conf);
  this.username = System.getProperty("user.name");
  this.jobACLs = aclsManager.constructJobACLs(conf);

  ThreadFactory threadFactory = new ThreadFactoryBuilder()
    .setNameFormat("Job Fail Wait Timeout Monitor #%d")
    .setDaemon(true)
    .build();
  this.executor = new ScheduledThreadPoolExecutor(1, threadFactory);

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  stateMachine = stateMachineFactory.make(this);
  this.forcedState  = forcedState;
  if(forcedDiagnostic != null) {
    this.diagnostics.add(forcedDiagnostic);
  }
  
  this.maxAllowedFetchFailuresFraction = conf.getFloat(
      MRJobConfig.MAX_ALLOWED_FETCH_FAILURES_FRACTION,
      MRJobConfig.DEFAULT_MAX_ALLOWED_FETCH_FAILURES_FRACTION);
  this.maxFetchFailuresNotifications = conf.getInt(
      MRJobConfig.MAX_FETCH_FAILURES_NOTIFICATIONS,
      MRJobConfig.DEFAULT_MAX_FETCH_FAILURES_NOTIFICATIONS);
}
 
Example 28
Source Project: big-c   Source File: TestTaskAttemptListenerImpl.java    License: Apache License 2.0 4 votes vote down vote up
public MockTaskAttemptListenerImpl(AppContext context,
    JobTokenSecretManager jobTokenSecretManager,
    RMHeartbeatHandler rmHeartbeatHandler) {
  super(context, jobTokenSecretManager, rmHeartbeatHandler, null);
}
 
Example 29
Source Project: big-c   Source File: TestTaskAttemptListenerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Test  (timeout=5000)
public void testGetTask() throws IOException {
  AppContext appCtx = mock(AppContext.class);
  JobTokenSecretManager secret = mock(JobTokenSecretManager.class); 
  RMHeartbeatHandler rmHeartbeatHandler =
      mock(RMHeartbeatHandler.class);
  TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class);
  MockTaskAttemptListenerImpl listener = 
    new MockTaskAttemptListenerImpl(appCtx, secret,
        rmHeartbeatHandler, hbHandler);
  Configuration conf = new Configuration();
  listener.init(conf);
  listener.start();
  JVMId id = new JVMId("foo",1, true, 1);
  WrappedJvmID wid = new WrappedJvmID(id.getJobId(), id.isMap, id.getId());

  // Verify ask before registration.
  //The JVM ID has not been registered yet so we should kill it.
  JvmContext context = new JvmContext();
  context.jvmId = id; 
  JvmTask result = listener.getTask(context);
  assertNotNull(result);
  assertTrue(result.shouldDie);

  // Verify ask after registration but before launch. 
  // Don't kill, should be null.
  TaskAttemptId attemptID = mock(TaskAttemptId.class);
  Task task = mock(Task.class);
  //Now put a task with the ID
  listener.registerPendingTask(task, wid);
  result = listener.getTask(context);
  assertNull(result);
  // Unregister for more testing.
  listener.unregister(attemptID, wid);

  // Verify ask after registration and launch
  //Now put a task with the ID
  listener.registerPendingTask(task, wid);
  listener.registerLaunchedTask(attemptID, wid);
  verify(hbHandler).register(attemptID);
  result = listener.getTask(context);
  assertNotNull(result);
  assertFalse(result.shouldDie);
  // Don't unregister yet for more testing.

  //Verify that if we call it again a second time we are told to die.
  result = listener.getTask(context);
  assertNotNull(result);
  assertTrue(result.shouldDie);

  listener.unregister(attemptID, wid);

  // Verify after unregistration.
  result = listener.getTask(context);
  assertNotNull(result);
  assertTrue(result.shouldDie);

  listener.stop();

  // test JVMID
  JVMId jvmid = JVMId.forName("jvm_001_002_m_004");
  assertNotNull(jvmid);
  try {
    JVMId.forName("jvm_001_002_m_004_006");
    Assert.fail();
  } catch (IllegalArgumentException e) {
    assertEquals(e.getMessage(),
        "TaskId string : jvm_001_002_m_004_006 is not properly formed");
  }

}
 
Example 30
Source Project: big-c   Source File: Application.java    License: Apache License 2.0 4 votes vote down vote up
public static String createDigest(byte[] password, String data)
    throws IOException {
  SecretKey key = JobTokenSecretManager.createSecretKey(password);
  return SecureShuffleUtils.hashFromString(data, key);
}